From 1cc00b717417c0a97cbfb42461e8d577d17b1438 Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Thu, 18 Aug 2022 18:29:04 +0800 Subject: [PATCH] Did the generation stuff change --- .../seibel/lod/core/a7/PlaceHolderQueue.java | 8 - .../generation/BlockingGenerationQueue.java | 26 -- .../core/a7/generation/GenerationQueue.java | 295 ++++++++-------- .../core/a7/generation/GenerationResult.java | 19 + .../lod/core/a7/level/DhClientLevel.java | 2 +- .../core/a7/level/DhClientServerLevel.java | 38 +- .../lod/core/a7/level/DhServerLevel.java | 2 +- .../core/a7/save/io/file/DataFileHandler.java | 63 +--- .../io/file/GeneratedDataFileHandler.java | 7 +- .../save/io/file/RemoteDataFileHandler.java | 43 +-- .../a7/save/io/render/RenderFileHandler.java | 8 - .../lod/core/a7/util/CombinableResult.java | 5 + .../a7/util/ConcurrentGenerationPosTree.java | 202 ----------- .../ConcurrentQuadCombinableProviderTree.java | 334 ++++++++++++++++++ 14 files changed, 529 insertions(+), 523 deletions(-) delete mode 100644 src/main/java/com/seibel/lod/core/a7/PlaceHolderQueue.java delete mode 100644 src/main/java/com/seibel/lod/core/a7/generation/BlockingGenerationQueue.java create mode 100644 src/main/java/com/seibel/lod/core/a7/generation/GenerationResult.java create mode 100644 src/main/java/com/seibel/lod/core/a7/util/CombinableResult.java delete mode 100644 src/main/java/com/seibel/lod/core/a7/util/ConcurrentGenerationPosTree.java create mode 100644 src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java diff --git a/src/main/java/com/seibel/lod/core/a7/PlaceHolderQueue.java b/src/main/java/com/seibel/lod/core/a7/PlaceHolderQueue.java deleted file mode 100644 index 19d6fd058..000000000 --- a/src/main/java/com/seibel/lod/core/a7/PlaceHolderQueue.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.seibel.lod.core.a7; - -import com.seibel.lod.core.a7.datatype.PlaceHolderRenderSource; - -@Deprecated -public interface PlaceHolderQueue { - void track(PlaceHolderRenderSource source); // Note: Implementation should only track a weak reference to the source. -} diff --git a/src/main/java/com/seibel/lod/core/a7/generation/BlockingGenerationQueue.java b/src/main/java/com/seibel/lod/core/a7/generation/BlockingGenerationQueue.java deleted file mode 100644 index c231e5766..000000000 --- a/src/main/java/com/seibel/lod/core/a7/generation/BlockingGenerationQueue.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.seibel.lod.core.a7.generation; - -import com.seibel.lod.core.a7.datatype.LodDataSource; -import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; -import com.seibel.lod.core.a7.pos.DhSectionPos; - -import java.util.concurrent.CompletableFuture; -import java.util.function.BiConsumer; -import java.util.function.Supplier; - -public class BlockingGenerationQueue { - final BiConsumer writeConsumer; - - - - - public BlockingGenerationQueue(BiConsumer writeConsumer) { - this.writeConsumer = writeConsumer; - } - - public CompletableFuture generate(DhSectionPos sectPos, Supplier creator) { - - } - - -} diff --git a/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java b/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java index 7ffbca850..5ca122435 100644 --- a/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java +++ b/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java @@ -1,13 +1,12 @@ package com.seibel.lod.core.a7.generation; import com.seibel.lod.core.a7.datatype.LodDataSource; -import com.seibel.lod.core.a7.datatype.PlaceHolderRenderSource; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullDataSource; import com.seibel.lod.core.a7.pos.DhBlockPos2D; import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.a7.pos.DhSectionPos; -import com.seibel.lod.core.a7.save.io.file.IDataSourceProvider; +import com.seibel.lod.core.a7.util.ConcurrentQuadCombinableProviderTree; import com.seibel.lod.core.a7.util.UncheckedInterruptedException; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.objects.DHChunkPos; @@ -15,112 +14,55 @@ import com.seibel.lod.core.util.LodUtil; import com.seibel.lod.core.util.gridList.ArrayGridList; import org.apache.logging.log4j.Logger; -import java.lang.ref.WeakReference; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedList; +import java.util.*; import java.util.concurrent.*; -import java.util.function.BiConsumer; - -public class GenerationQueue { - static class Request implements Comparable { - long cachedDist; - final DhSectionPos sectionPos; - Request(DhSectionPos sectionPos) { - this.sectionPos = sectionPos; - } - @Override - public int compareTo(Request o) { - return Long.compare(cachedDist, o.cachedDist); - } - @Override - public int hashCode() { - return sectionPos.hashCode(); - } - @Override - public boolean equals(Object obj) { - if (obj instanceof Request) { - return sectionPos.equals(((Request) obj).sectionPos); - } - return false; - } - } - - - - - - - - public CompletableFuture generate(DhSectionPos sectionPos) { - - - return CompletableFuture.supplyAsync(() -> { - try { - return generate0(sectionPos); - } catch (Exception e) { - throw new CompletionException(e); - } - }); - } - +public class GenerationQueue implements AutoCloseable { + final ConcurrentQuadCombinableProviderTree cqcpTree = new ConcurrentQuadCombinableProviderTree<>(); + IGenerator generator = null; private final Logger logger = DhLoggerBuilder.getLogger(); - DhBlockPos2D lastPlayerPos = new DhBlockPos2D(0, 0); - final ConcurrentHashMap> trackers = new ConcurrentHashMap<>(); - final BiConsumer writeConsumer; - final HashSet> inProgressSections = new HashSet<>(); + private final ConcurrentHashMap> taskMap = new ConcurrentHashMap<>(); + private final LinkedList> inProgress = new LinkedList<>(); - public GenerationQueue(BiConsumer writeConsumer) { - this.writeConsumer = writeConsumer; - } - - public void track(PlaceHolderRenderSource source) { - //logger.info("Tracking source {} at {}", source, source.getSectionPos()); - trackers.put(source.getSectionPos(), new WeakReference<>(source)); - } - - private void update() { - LinkedList toRemove = new LinkedList<>(); - for (DhSectionPos pos : trackers.keySet()) { - WeakReference ref = trackers.get(pos); - if (ref.get() == null) { - toRemove.add(pos); - } - } - for (DhSectionPos pos : toRemove) { - trackers.remove(pos); - } - } - - //FIXME: Do optimizations on polling closest to player. (Currently its a O(n) search!) - //FIXME: Do not return sections that is already being generated. - //FIXME: Optimize the checks for inProgressSections. - private DhSectionPos pollClosest(DhBlockPos2D playerPos) { - update(); - DhSectionPos closest = null; - long closestDist = Long.MAX_VALUE; - for (DhSectionPos pos : trackers.keySet()) { - if (inProgressSections.contains(pos)) { - continue; - } - long distSqr = pos.getCenter().getCenter().distSquared(playerPos); - if (distSqr < closestDist) { - closest = pos; - closestDist = distSqr; - } - } - if (closest != null) inProgressSections.add(closest); - return closest; - } - - public void doGeneration(IGenerator generator) { - if (generator == null) return; + public GenerationQueue() {} + public void pollAndStartClosest(DhBlockPos2D targetPos) { + if (generator == null) throw new IllegalStateException("generator is null"); if (generator.isBusy()) return; + DhLodPos closest = null; + long closestDist = Long.MAX_VALUE; + for (DhLodPos key : taskMap.keySet()) { + long dist = key.getCenter().distSquared(targetPos); + if (dist < closestDist) { + closest = key; + closestDist = dist; + } + } + if (closest != null) { + CompletableFuture future = taskMap.remove(closest); + startFuture(closest, future); + } + } - DhSectionPos pos = pollClosest(lastPlayerPos); - if (pos == null) return; + public void setGenerator(IGenerator generator) { + LodUtil.assertTrue(generator != null); + LodUtil.assertTrue(this.generator == null); + this.generator = generator; + } + public void removeGenerator() { + LodUtil.assertTrue(generator != null); + this.generator = null; + inProgress.forEach(f -> f.cancel(true)); + inProgress.clear(); + } + private CompletableFuture createFuture(DhLodPos pos) { + CompletableFuture future = new CompletableFuture<>(); + CompletableFuture swapped = taskMap.put(pos, future); + LodUtil.assertTrue(swapped == null); + return future; + } + + private void startFuture(DhLodPos pos, CompletableFuture resultFuture) { byte dataDetail = generator.getDataDetail(); byte minGenGranularity = generator.getMinGenerationGranularity(); byte maxGenGranularity = generator.getMaxGenerationGranularity(); @@ -134,18 +76,18 @@ public class GenerationQueue { byte granularity; int count; DHChunkPos chunkPosMin; - if (pos.sectionDetail < minUnitDetail) { + if (pos.detail < minUnitDetail) { granularity = minGenGranularity; count = 1; - chunkPosMin = new DHChunkPos(pos.getSectionBBoxPos().convertUpwardsTo(minUnitDetail).getCorner()); - } else if (pos.sectionDetail > maxUnitDetail) { + chunkPosMin = new DHChunkPos(pos.convertUpwardsTo(minUnitDetail).getCorner()); + } else if (pos.detail > maxUnitDetail) { granularity = maxGenGranularity; - count = 1 << (pos.sectionDetail - maxUnitDetail); - chunkPosMin = new DHChunkPos(pos.getCorner().getCorner()); + count = 1 << (pos.detail - maxUnitDetail); + chunkPosMin = new DHChunkPos(pos.getCorner()); } else { - granularity = (byte) (pos.sectionDetail - dataDetail); + granularity = (byte) (pos.detail - dataDetail); count = 1; - chunkPosMin = new DHChunkPos(pos.getCorner().getCorner()); + chunkPosMin = new DHChunkPos(pos.getCorner()); } assert granularity >= minGenGranularity && granularity <= maxGenGranularity; assert count > 0; @@ -154,27 +96,27 @@ public class GenerationQueue { int perCallChunksWidth = 1 << (granularity - 4); final byte sectionDetail = (byte) (dataDetail + FullDataSource.SECTION_SIZE_OFFSET); - ArrayList> futures = new ArrayList<>(count*count); + ArrayList>> futures = new ArrayList<>(count*count); for (int dx = 0; dx < count; dx++) { for (int dz = 0; dz < count; dz++) { // TODO: Unroll this loop to yield when generator is busy. DHChunkPos subCallChunkPosMin = new DHChunkPos(chunkPosMin.x + dx * perCallChunksWidth, chunkPosMin.z + dz * perCallChunksWidth); CompletableFuture> dataFuture = generator.generate(subCallChunkPosMin, granularity); - futures.add(dataFuture.whenComplete((data, ex) -> { + futures.add(dataFuture.handle((data, ex) -> { if (ex != null) { if (ex instanceof CompletionException) { ex = ex.getCause(); } - if (ex instanceof InterruptedException) return; // Ignore interrupted exceptions. - if (ex instanceof UncheckedInterruptedException) return; // Ignore unchecked interrupted exceptions. + if (ex instanceof InterruptedException) return null; // Ignore interrupted exceptions. + if (ex instanceof UncheckedInterruptedException) return null; // Ignore unchecked interrupted exceptions. logger.error("Error generating data for section {}", pos, ex); - return; + return null; } - assert data != null; + LodUtil.assertTrue(data != null); if (data.gridSize < (1 << (granularity-4))) { logger.error( "Generator at {} returned {} by {} chunks but requested granularity was {}, which expect at least {} by {} chunks! ", pos, data.gridSize, data.gridSize, granularity, perCallChunksWidth, perCallChunksWidth); - return; + return null; } DhLodPos minSectPos = new DhLodPos((byte)(dataDetail+4), data.getFirst().x, data.getFirst().z).convertUpwardsTo(sectionDetail); @@ -186,49 +128,88 @@ public class GenerationQueue { logger.info("Writing {} by {} chunks (at {}) with data detail {} to {} by {} sections (at {})", data.gridSize, data.gridSize, subCallChunkPosMin, dataDetail, sectionCount, sectionCount, minSectPos); - - data.forEachPos((x,z) -> { - ChunkSizedData chunkData = data.get(x,z); - DhLodPos chunkDataPos = new DhLodPos((byte)(chunkData.dataDetail + 4), chunkData.x, chunkData.z); - if (pos.getSectionBBoxPos().overlaps(chunkDataPos)) - writeConsumer.accept(pos, chunkData); - //DhSectionPos sectionPos = new DhSectionPos(chunkDataPos.detail, chunkDataPos.x, chunkDataPos.z); - //logger.info("Writing chunk {} with data detail {} to section {}", - // new DhLodPos((byte)(chunkData.dataDetail + 4), chunkData.x, chunkData.z), - // dataDetail, sectionPos); - }); -// -// for (int dsx = 0; dsx < sectionCount; dsx++) { -// for (int dsz = 0; dsz < sectionCount; dsz++) { -// WeakReference ref = trackers.remove(new DhSectionPos( -// sectionDetail, minSectPos.x + dsx, minSectPos.z + dsz)); -// if (ref == null) return; // No placeholder there, so no need to trigger a refresh on it. -// PlaceHolderRenderSource source = ref.get(); -// if (source == null) return; // Same as above. -// source.markInvalid(); // Mark the placeholder as invalid, so it will be refreshed on next lodTree update. -// } -// } - }).exceptionally(ex -> { - if (ex instanceof CompletionException) { - ex = ex.getCause(); - } - if (ex instanceof InterruptedException) return null; // Ignore interrupted exceptions. - if (ex instanceof UncheckedInterruptedException) return null; // Ignore unchecked interrupted exceptions. - logger.error("Error generating data for {} by {} chunks (at {}) with data detail {}", - perCallChunksWidth, perCallChunksWidth, subCallChunkPosMin, dataDetail, ex); - return null; - }).thenRun(()->{})); // Convert to a CompletableFuture. + return data; + })); } } - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenRun(() -> { - //try { - //Thread.sleep(10000); // FIXME: Only for current debug testing. REMOVE THIS! - //} catch (InterruptedException ignored) {} - WeakReference ref = trackers.remove(pos); - if (ref == null) return; // No placeholder there, so no need to trigger a refresh on it. - PlaceHolderRenderSource source = ref.get(); - if (source == null) return; // Same as above. - source.markInvalid(); // Mark the placeholder as invalid, so it will be refreshed on next lodTree update. - }); + inProgress.add( + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenApply((v) -> { + GenerationResult result = new GenerationResult(); + for (CompletableFuture> future : futures) { + Collection data = future.join(); + if (data == null) continue; + result.dataList.addAll(data); + } + return result; + }).handle((r, e) -> { + if (e!=null) resultFuture.completeExceptionally(e); else resultFuture.complete(r); + return null; + }) + ); } + + public CompletableFuture generate(DhSectionPos sectionPos) { + DhLodPos lodPos = sectionPos.getSectionBBoxPos(); + return cqcpTree.createOrUseExisting(lodPos, this::createFuture).thenApply( + (result) -> { + if (result == null || result.dataList.isEmpty()) return FullDataSource.createEmpty(sectionPos); + FullDataSource newSource = FullDataSource.createEmpty(sectionPos); + for (ChunkSizedData data : result.dataList) { + newSource.update(data); + } + return newSource; + }); + } + + @Override + public void close() { + //TODO + + } + +// +// DhBlockPos2D lastPlayerPos = new DhBlockPos2D(0, 0); +// final ConcurrentHashMap> trackers = new ConcurrentHashMap<>(); +// final BiConsumer writeConsumer; +// final HashSet> inProgressSections = new HashSet<>(); + +// +// public void track(PlaceHolderRenderSource source) { +// //logger.info("Tracking source {} at {}", source, source.getSectionPos()); +// trackers.put(source.getSectionPos(), new WeakReference<>(source)); +// } +// +// private void update() { +// LinkedList toRemove = new LinkedList<>(); +// for (DhSectionPos pos : trackers.keySet()) { +// WeakReference ref = trackers.get(pos); +// if (ref.get() == null) { +// toRemove.add(pos); +// } +// } +// for (DhSectionPos pos : toRemove) { +// trackers.remove(pos); +// } +// } + +// //FIXME: Do optimizations on polling closest to player. (Currently its a O(n) search!) +// //FIXME: Do not return sections that is already being generated. +// //FIXME: Optimize the checks for inProgressSections. +// private DhSectionPos pollClosest(DhBlockPos2D playerPos) { +// update(); +// DhSectionPos closest = null; +// long closestDist = Long.MAX_VALUE; +// for (DhSectionPos pos : trackers.keySet()) { +// if (inProgressSections.contains(pos)) { +// continue; +// } +// long distSqr = pos.getCenter().getCenter().distSquared(playerPos); +// if (distSqr < closestDist) { +// closest = pos; +// closestDist = distSqr; +// } +// } +// if (closest != null) inProgressSections.add(closest); +// return closest; +// } } diff --git a/src/main/java/com/seibel/lod/core/a7/generation/GenerationResult.java b/src/main/java/com/seibel/lod/core/a7/generation/GenerationResult.java new file mode 100644 index 000000000..1685eabaa --- /dev/null +++ b/src/main/java/com/seibel/lod/core/a7/generation/GenerationResult.java @@ -0,0 +1,19 @@ +package com.seibel.lod.core.a7.generation; + +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; +import com.seibel.lod.core.a7.util.CombinableResult; + +import java.util.ArrayList; + +public class GenerationResult implements CombinableResult { + public final ArrayList dataList = new ArrayList<>(); + + @Override + public GenerationResult combineWith(GenerationResult b, GenerationResult c, GenerationResult d) { + dataList.ensureCapacity(dataList.size() + b.dataList.size() + c.dataList.size() + d.dataList.size()); + dataList.addAll(b.dataList); + dataList.addAll(c.dataList); + dataList.addAll(d.dataList); + return this; + } +} diff --git a/src/main/java/com/seibel/lod/core/a7/level/DhClientLevel.java b/src/main/java/com/seibel/lod/core/a7/level/DhClientLevel.java index 0e0333a4a..34a54204d 100644 --- a/src/main/java/com/seibel/lod/core/a7/level/DhClientLevel.java +++ b/src/main/java/com/seibel/lod/core/a7/level/DhClientLevel.java @@ -38,7 +38,7 @@ public class DhClientLevel implements IClientLevel { this.save = save; save.getDataFolder(level).mkdirs(); save.getRenderCacheFolder(level).mkdirs(); - dataFileHandler = new RemoteDataFileHandler(); + dataFileHandler = new RemoteDataFileHandler(this, save.getDataFolder(level)); renderFileHandler = new RenderFileHandler(dataFileHandler, this, save.getRenderCacheFolder(level)); tree = new LodQuadTree(this, Config.Client.Graphics.Quality.lodChunkRenderDistance.get()*16, MC_CLIENT.getPlayerBlockPos().x, MC_CLIENT.getPlayerBlockPos().z, renderFileHandler); diff --git a/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java b/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java index d47e91b3a..a881d7b2f 100644 --- a/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java +++ b/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java @@ -3,6 +3,7 @@ package com.seibel.lod.core.a7.level; import com.seibel.lod.core.a7.generation.GenerationQueue; import com.seibel.lod.core.a7.generation.IGenerator; import com.seibel.lod.core.a7.render.LodQuadTree; +import com.seibel.lod.core.a7.save.io.file.GeneratedDataFileHandler; import com.seibel.lod.core.a7.util.FileScanner; import com.seibel.lod.core.a7.save.io.file.DataFileHandler; import com.seibel.lod.core.a7.save.io.render.RenderFileHandler; @@ -38,14 +39,15 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { public IClientLevelWrapper clientLevel; public a7LodRenderer renderer = null; public LodQuadTree tree = null; - public IGenerator worldGenerator = null; + public BatchGenerator worldGenerator = null; public DhClientServerLevel(LocalSaveStructure save, IServerLevelWrapper level) { this.serverLevel = level; this.save = save; save.getDataFolder(level).mkdirs(); save.getRenderCacheFolder(level).mkdirs(); - dataFileHandler = new DataFileHandler(this, save.getDataFolder(level)); + generationQueue = new GenerationQueue(); + dataFileHandler = new GeneratedDataFileHandler(this, save.getDataFolder(level), generationQueue); FileScanner.scanFile(save, serverLevel, dataFileHandler, null); LOGGER.info("Started DHLevel for {} with saves at {}", level, save); } @@ -61,6 +63,7 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { public void serverTick() { //TODO Update network packet and stuff or state or etc.. } + public void startRenderer(IClientLevelWrapper clientLevel) { LOGGER.info("Starting renderer for {}", this); if (renderBufferHandler != null || this.clientLevel != null) { @@ -68,13 +71,10 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { return; } this.clientLevel = clientLevel; - - // FIXME: This A need B and B need A messes needs to be reworked! + // TODO: Make a registry for generators for modding support. + worldGenerator = new BatchGenerator(this); + generationQueue.setGenerator(worldGenerator); renderFileHandler = new RenderFileHandler(dataFileHandler, this, save.getRenderCacheFolder(serverLevel)); - final RenderFileHandler f_renderFileHandler = renderFileHandler; - generationQueue = new GenerationQueue(f_renderFileHandler::write); - renderFileHandler.setPlaceHolderQueue(generationQueue); - tree = new LodQuadTree(this, Config.Client.Graphics.Quality.lodChunkRenderDistance.get()*16, MC_CLIENT.getPlayerBlockPos().x, MC_CLIENT.getPlayerBlockPos().z, renderFileHandler); renderBufferHandler = new RenderBufferHandler(tree); @@ -103,10 +103,16 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { tree = null; renderBufferHandler.close(); renderBufferHandler = null; - generationQueue = null; renderFileHandler.flushAndSave(); //Ignore the completion feature so that this action is async renderFileHandler.close(); renderFileHandler = null; + generationQueue.removeGenerator(); + try { + worldGenerator.close(); + } catch (Exception e) { + LOGGER.error("Error closing world generator", e); + } + worldGenerator = null; } @Override @@ -139,11 +145,9 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { return renderFileHandler == null ? dataFileHandler.flushAndSave() : renderFileHandler.flushAndSave(); //Note: saving renderFileHandler will also save the dataFileHandler. } - - private BatchGenerator batchGenerator = null; @Override public void close() { - if (batchGenerator != null) batchGenerator.close(); + if (worldGenerator != null) worldGenerator.close(); if (renderer != null) renderer.close(); if (tree != null) tree.close(); if (renderBufferHandler != null) renderBufferHandler.close(); @@ -155,14 +159,10 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { @Override public void doWorldGen() { - if (worldGenerator == null) { - // TODO: Make a registry for generators for modding support. - batchGenerator = new BatchGenerator(this); - worldGenerator = batchGenerator; - } else { - batchGenerator.update(); + if (worldGenerator != null) { + worldGenerator.update(); if (generationQueue != null) - generationQueue.doGeneration(batchGenerator); + generationQueue.pollAndStartClosest(new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos())); } } diff --git a/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java b/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java index 6946b40df..c184b76ec 100644 --- a/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java +++ b/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java @@ -20,7 +20,7 @@ public class DhServerLevel implements IServerLevel { this.save = save; this.level = level; save.getDataFolder(level).mkdirs(); - dataFileHandler = new DataFileHandler(this, save.getDataFolder(level)); + dataFileHandler = new DataFileHandler(this, save.getDataFolder(level), null); //FIXME: GenerationQueue FileScanner.scanFile(save, level, dataFileHandler, null); LOGGER.info("Started DHLevel for {} with saves at {}", level, save); } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java b/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java index a68ce910b..c2373cec5 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java @@ -4,6 +4,7 @@ import com.google.common.collect.HashMultimap; import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullDataSource; +import com.seibel.lod.core.a7.level.ILevel; import com.seibel.lod.core.a7.level.IServerLevel; import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.a7.pos.DhSectionPos; @@ -28,14 +29,14 @@ public class DataFileHandler implements IDataSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); final ExecutorService fileReaderThread = LodUtil.makeThreadPool(4, "FileReaderThread"); final ConcurrentHashMap files = new ConcurrentHashMap<>(); - final IServerLevel level; + final ILevel level; final File saveDir; AtomicInteger topDetailLevel = new AtomicInteger(-1); final int minDetailLevel = FullDataSource.SECTION_SIZE_OFFSET; final Function> dataSourceCreator; - public DataFileHandler(IServerLevel level, File saveRootDir, + public DataFileHandler(ILevel level, File saveRootDir, Function> dataSourceCreator) { this.saveDir = saveRootDir; this.level = level; @@ -139,15 +140,6 @@ public class DataFileHandler implements IDataSourceProvider { return metaFile.loadOrGetCached(fileReaderThread); } - // This prevents new higher detail levels from being created by not updating the topDetailLevel. - private CompletableFuture readWithoutUpdate(DhSectionPos pos) { - DataMetaFile metaFile = files.get(pos); - if (metaFile == null) { - return CompletableFuture.completedFuture(null); - } - return metaFile.loadOrGetCached(fileReaderThread); - } - /* * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ @@ -163,55 +155,6 @@ public class DataFileHandler implements IDataSourceProvider { if (metaFile != null) { // Fast path: if there is a file for this section, just write to it. metaFile.addToWriteQueue(chunkData); } -/* else if (sectionPos.sectionDetail <= minDetailLevel) { - File file = computeDefaultFilePath(sectionPos); - //FIXME: Handle file already exists issue. Possibly by renaming the file. - LodUtil.assertTrue(!file.exists(), "File {} already exist for path {}", file, sectionPos); - CompletableFuture gen = new CompletableFuture<>(); - DataMetaFile newMetaFile = new DataMetaFile(level, file, sectionPos, gen); - metaFile = files.putIfAbsent(sectionPos, newMetaFile); // This is a CAS with expected null value. - if (metaFile == null) { - newMetaFile.addToWriteQueue(chunkData); - CompletableFuture.runAsync(() -> gen.complete(FullDataSource.createEmpty(sectionPos)), fileReaderThread) - .exceptionally((e) -> { - gen.completeExceptionally(e); - return null; - }); - } else { - metaFile.addToWriteQueue(chunkData); - gen.cancel(true); - } - } else { - File file = computeDefaultFilePath(sectionPos); - //FIXME: Handle file already exists issue. Possibly by renaming the file. - LodUtil.assertTrue(!file.exists(), "File {} already exist for path {}", file, sectionPos); - CompletableFuture gen = new CompletableFuture<>(); - DataMetaFile newMetaFile = new DataMetaFile(level, file, sectionPos, gen); - metaFile = files.putIfAbsent(sectionPos, newMetaFile); // This is a CAS with expected null value. - if (metaFile == null) { - newMetaFile.addToWriteQueue(chunkData); - // Create future that generate downsized file - CompletableFuture subChunk0 = readWithoutUpdate(sectionPos.getChild(0)); - CompletableFuture subChunk1 = readWithoutUpdate(sectionPos.getChild(1)); - CompletableFuture subChunk2 = readWithoutUpdate(sectionPos.getChild(2)); - CompletableFuture subChunk3 = readWithoutUpdate(sectionPos.getChild(3)); - CompletableFuture.allOf(subChunk0, subChunk1, subChunk2, subChunk3) - .thenAccept(v -> - gen.complete(FullDataSource.createFromLower(sectionPos, new FullDataSource[]{ - (FullDataSource) subChunk0.join(), - (FullDataSource) subChunk1.join(), - (FullDataSource) subChunk2.join(), - (FullDataSource) subChunk3.join() - })) - ).exceptionally((e) -> { - gen.completeExceptionally(e); - return null; - }); - } else { - metaFile.addToWriteQueue(chunkData); - gen.cancel(true); - } - }*/ if (sectionPos.sectionDetail <= topDetailLevel.get()) { recursiveWrite(sectionPos.getParent(), chunkData); } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java b/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java index b7d8ee68d..56fe8b8de 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java @@ -1,17 +1,12 @@ package com.seibel.lod.core.a7.save.io.file; -import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.generation.GenerationQueue; import com.seibel.lod.core.a7.level.IServerLevel; -import com.seibel.lod.core.a7.pos.DhSectionPos; import java.io.File; -import java.util.concurrent.CompletableFuture; -import java.util.function.Function; public class GeneratedDataFileHandler extends DataFileHandler { - public GeneratedDataFileHandler(IServerLevel level, File saveRootDir, GenerationQueue queue) { - super(level, saveRootDir, dataSourceCreator); + super(level, saveRootDir, queue::generate); } } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java b/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java index f25704d27..43d604dcc 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java @@ -1,42 +1,15 @@ package com.seibel.lod.core.a7.save.io.file; -import com.seibel.lod.core.a7.datatype.LodDataSource; -import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; -import com.seibel.lod.core.a7.datatype.full.FullFormat; -import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.a7.level.ILevel; +import com.seibel.lod.core.util.LodUtil; import java.io.File; -import java.util.Collection; -import java.util.concurrent.CompletableFuture; - -public class RemoteDataFileHandler implements IDataSourceProvider { - @Override - public void addScannedFile(Collection detectedFiles) { - - } - - @Override - public CompletableFuture read(DhSectionPos pos) { - return null; - } - - @Override - public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { - - } - - @Override - public CompletableFuture flushAndSave() { - return null; - } - - @Override - public boolean isCacheValid(DhSectionPos sectionPos, long timestamp) { - return false; - } - - @Override - public void close() throws Exception { +public class RemoteDataFileHandler extends DataFileHandler { + public RemoteDataFileHandler(ILevel level, File saveRootDir) { + super(level, saveRootDir, (pos) -> { + LodUtil.assertNotReach("TODO"); + return null; + }); } } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java b/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java index fa8c53e6f..7d8acdc7a 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java @@ -1,7 +1,6 @@ package com.seibel.lod.core.a7.save.io.render; import com.google.common.collect.HashMultimap; -import com.seibel.lod.core.a7.PlaceHolderQueue; import com.seibel.lod.core.a7.datatype.PlaceHolderRenderSource; import com.seibel.lod.core.a7.datatype.LodRenderSource; import com.seibel.lod.core.a7.datatype.column.ColumnRenderSource; @@ -29,8 +28,6 @@ public class RenderFileHandler implements IRenderSourceProvider { final IClientLevel level; final File saveDir; final IDataSourceProvider dataSourceProvider; - @Nullable - PlaceHolderQueue placeHolderQueue = null; public RenderFileHandler(IDataSourceProvider sourceProvider, IClientLevel level, File saveRootDir) { this.dataSourceProvider = sourceProvider; @@ -38,10 +35,6 @@ public class RenderFileHandler implements IRenderSourceProvider { this.saveDir = saveRootDir; } - public void setPlaceHolderQueue(@Nullable PlaceHolderQueue queue) { - this.placeHolderQueue = queue; - } - /* * Caller must ensure that this method is called only once, * and that this object is not used before this method is called. @@ -122,7 +115,6 @@ public class RenderFileHandler implements IRenderSourceProvider { } if (render != null) return render; PlaceHolderRenderSource placeHolder = new PlaceHolderRenderSource(pos); - if (placeHolderQueue != null) placeHolderQueue.track(placeHolder); return placeHolder; } ); diff --git a/src/main/java/com/seibel/lod/core/a7/util/CombinableResult.java b/src/main/java/com/seibel/lod/core/a7/util/CombinableResult.java new file mode 100644 index 000000000..6acfb7acb --- /dev/null +++ b/src/main/java/com/seibel/lod/core/a7/util/CombinableResult.java @@ -0,0 +1,5 @@ +package com.seibel.lod.core.a7.util; + +public interface CombinableResult { + T combineWith(T b, T c, T d); +} diff --git a/src/main/java/com/seibel/lod/core/a7/util/ConcurrentGenerationPosTree.java b/src/main/java/com/seibel/lod/core/a7/util/ConcurrentGenerationPosTree.java deleted file mode 100644 index 94b29a4b3..000000000 --- a/src/main/java/com/seibel/lod/core/a7/util/ConcurrentGenerationPosTree.java +++ /dev/null @@ -1,202 +0,0 @@ -package com.seibel.lod.core.a7.util; - -import com.seibel.lod.core.a7.pos.DhLodPos; -import com.seibel.lod.core.util.Atomics; -import com.seibel.lod.core.util.LodUtil; -import com.sun.jna.platform.unix.X11; -import org.apache.commons.lang3.NotImplementedException; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.AtomicReferenceArray; -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -interface CombinableResult { - T combineWith(T b, T c, T d); -} - -public class ConcurrentGenerationPosTree> { - public class Node { - private final DhLodPos pos; - public final AtomicReference> future; - public final AtomicReferenceArray children = new AtomicReferenceArray<>(4); - private Node(DhLodPos pos, CompletableFuture future) { - this.pos = pos; - this.future = new AtomicReference<>(future); - } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Node node = (Node) o; - return pos.equals(node.pos); - } - @Override - public int hashCode() { - return pos.hashCode(); - } - } - - class RootMap { - private final ConcurrentSkipListMap roots = new ConcurrentSkipListMap<>(); - private final int topLevel; - RootMap(int topLevel) { - this.topLevel = topLevel; - } - Node get(DhLodPos pos) { - return roots.get(pos); - } - Node swap(DhLodPos pos, Node newRoot) { - return roots.put(pos, newRoot); - } - Node compareNullAndExchange(DhLodPos pos, Node newRoot) { - return roots.putIfAbsent(pos, newRoot); - } - boolean compareNullAndSet(DhLodPos pos, Node newRoot) { - Node oldRoot = roots.putIfAbsent(pos, newRoot); - return oldRoot == null; - } - Node setIfNullAndGet(DhLodPos pos, Node newRoot) { - Node oldRoot = roots.putIfAbsent(pos, newRoot); - return oldRoot == null ? newRoot : oldRoot; - } - Node swapNull(DhLodPos pos) { - return roots.remove(pos); - } - - } - private final AtomicReference rootMap = new AtomicReference<>(new RootMap(0)); - - - public ConcurrentGenerationPosTree() {} - @Override - public int hashCode() { - throw new NotImplementedException(); - } - @Override - public boolean equals(Object obj) { - throw new NotImplementedException(); - } - @Override - protected Object clone() throws CloneNotSupportedException { - throw new NotImplementedException(); - } - @Override - public String toString() { - throw new NotImplementedException(); - } - - // Atomically update and get the generation future - private CompletableFuture checkAndMakeFuture(Node node, BiConsumer> allNullCompleter) { - CompletableFuture future = new CompletableFuture<>(); - CompletableFuture casValue = Atomics.compareAndExchange(node.future, null, future); - if (casValue != null) { // cas failed. Existing future. Return it. - return future; - } - // Next, we need to make the future completable. - // We first check for each child connection if it exists. If it does, we store it for a later 'allOf'. - boolean allNull = true; - @SuppressWarnings("unchecked") - CompletableFuture[] childFutures = new CompletableFuture[4]; - for (int i = 0; i < 4; i++) { - Node nextChild = node.children.get(i); - if (nextChild != null) { // child node exists. Recursively make or get the child's future. - allNull = false; - childFutures[i] = checkAndMakeFuture(nextChild, allNullCompleter); - } - } - if (allNull) { // all children are null. We can then just run the allNullCompleter in this node. - allNullCompleter.accept(node.pos, future); - } else { // some children exist. We need to wait for some or all of them to complete. - // But before that, we need to create the children node where they are missing. - for (int i = 0; i < 4; i++) { - if (childFutures[i] == null) { - CompletableFuture newChildFuture = new CompletableFuture<>(); - node.children.set(i, new Node(node.pos.getChild(i), newChildFuture)); - // Since the child is new, we can be sure that it doesn't have any children. - // So, we need to make the new child's future completable by running the allNullCompleter. - // (The above relies on the fact that we did a CAS on the beginning of this method, - // which means that we have unique access to the node and it's links to the children, and that - // no other thread can be concurrently modifying its links) - allNullCompleter.accept(node.pos.getChild(i), newChildFuture); - } - } - // Now, we can wait for all the child futures to complete, and then complete this node's future with - // the combined result of all child futures. - CompletableFuture.allOf(childFutures).thenRun(() -> - future.complete(childFutures[0].join().combineWith( - childFutures[1].join(), childFutures[2].join(), childFutures[3].join()))); - } - return future; - } - - public CompletableFuture createOrUseExisting(DhLodPos pos, BiConsumer> completer) { - RootMap map = rootMap.get(); - if (map.topLevel == pos.detail) { - CompletableFuture future = new CompletableFuture<>(); - Node newNode = new Node(pos, future); - Node cas = map.compareNullAndExchange(pos, newNode); - if (cas == null) { // cas succeeded. No existing overlapping node in same detail level. - // Since any lower level nodes should have upper level nodes as parent up to the top level, - // and that there are no same level nodes, we can assume that the new node does not overlap any existing nodes. - // Therefore, we can apply the completer function to the new node, and return the future. - completer.accept(pos, future); - return future; - } else { // cas failed. Existing overlapping node. - // Run the checkAndMakeFuture method on the existing node to update and get the generation future. - return checkAndMakeFuture(cas, completer); - } - } else if (map.topLevel > pos.detail) { - // We need to traverse down the tree with the following rules during the traversal: - // 1. If the next node is not null and has a future, halt and return that future. - // 2. If the next node is not null with no future, continue traversing down the tree. - // 3. if the next node is null, create a new node and CompareExchange it into the current node, and run rule 1/2. - // Note that DO NOT assume that all subsequent nodes will fall into case 3, as someone else can concurrently - // use and modify the newly created node! - - // To start, just treat the rootMap as the... well, root, and it's content as the children node. - // We can then traverse down the tree until we reach the target node or hit the 1st case and return prematurely. - - // First iteration: - Node currentNode; - Node childNode = map.setIfNullAndGet(pos.convertUpwardsTo((byte) map.topLevel), new Node(pos, null)); // rule 3: if null, create a new node. - CompletableFuture future = childNode.future.get(); - if (future != null) { // rule 1: if future is not null, halt and return the future. - return future; - } else { // rule 2: if future is null, continue traversing down the tree. - currentNode = childNode; - - // Second and subsequent iterations: - while (currentNode.pos.detail > pos.detail) { - Node newNode = new Node(pos.convertUpwardsTo((byte)(currentNode.pos.detail-1)), null); - childNode = Atomics.compareAndSetThenGet(currentNode.children, - newNode.pos.getChildIndexOfParent(), null, newNode); // rule 3: if null, create a new node. - if (childNode == null) { // rule 1: if future is not null, halt and return the future. - return newNode.future.get(); - } else { // rule 2: if future is null, continue traversing down the tree. - currentNode = childNode; - } - } - } - // At this point, we have reached the target node. - LodUtil.assertTrue(currentNode.pos.equals(pos)); - // We can now run the checkAndMakeFuture method on the target node to update and get the generation future. - return checkAndMakeFuture(currentNode, completer); // Technically, this will rerun the 1st rule. But code is cleaner this way. - } else { // map.topLevel < pos.detail - // Now, this is the complex case. We need to rebase the tree to the higher detail level. - // For now, this implementation will do a lock based version. However, I will figure out a way to do this without a lock. - - // Before we do anything, we ... - - // TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - return null; - - } - } -} diff --git a/src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java b/src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java new file mode 100644 index 000000000..8f184c922 --- /dev/null +++ b/src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java @@ -0,0 +1,334 @@ +package com.seibel.lod.core.a7.util; + +import com.seibel.lod.core.a7.pos.DhLodPos; +import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.util.Atomics; +import com.seibel.lod.core.util.LodUtil; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.logging.log4j.Logger; + +import java.lang.ref.WeakReference; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; + +public class ConcurrentQuadCombinableProviderTree> { + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + public static class Node { + private final DhLodPos pos; + public final AtomicReference> future; + // The child node is stored as a weak reference so that it can be garbage collected when that node's future is completed + // and which then releases the hold on that node, thus allowing automatic garbage collection. + public final AtomicReferenceArray>> children = new AtomicReferenceArray<>(4); + @SuppressWarnings("unused") + AtomicReference> parent = null; // This is only used to ensure that the parent is not garbage collected before the child. + private Node(DhLodPos pos, CompletableFuture future) { + this.pos = pos; + this.future = new AtomicReference<>(future); + } + private Node(DhLodPos pos, CompletableFuture future, Node parent) { + this.pos = pos; + this.future = new AtomicReference<>(future); + this.parent = new AtomicReference<>(parent); + } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Node node = (Node) o; + return pos.equals(node.pos); + } + @Override + public int hashCode() { + return pos.hashCode(); + } + public Node setIfNullAndGet(int childIndex, Node newChild) { + WeakReference> newRef = new WeakReference<>(newChild); + WeakReference> oldRef; + do { + oldRef = Atomics.compareAndExchange(children, childIndex, null, newRef); + if (oldRef == null) return newChild; // CompareAndExchange succeeded + Node oldNode = oldRef.get(); + if (oldNode != null) return oldNode; // CompareAndExchange failed with old node not null + // Otherwise, the old node weak reference is null. + } while (!children.compareAndSet(childIndex, oldRef, newRef)); // If this cas fails, then try again. (Some other thread beat us to it.) + return newChild; // If we get here, then we successfully replaced the old node weak reference with the new one. + } + } + + static class RootMap { + private final ConcurrentHashMap>> roots = new ConcurrentHashMap<>(); + private final int topLevel; + + RootMap(int topLevel) { + this.topLevel = topLevel; + } + + public int getTopLevel() { + return topLevel; + } + public Node get(DhLodPos pos) { + WeakReference> ref = roots.get(pos); + return ref == null ? null : ref.get(); + } + public Node compareNullAndExchange(DhLodPos pos, Node newRoot) { + WeakReference> newRef = new WeakReference<>(newRoot); + WeakReference> oldRef; + do { + oldRef = roots.putIfAbsent(pos, newRef); + if (oldRef == null) return null; // putIfAbsent succeeded + Node oldRoot = oldRef.get(); + if (oldRoot != null) return oldRoot; // putIfAbsent failed with old root not null + // Otherwise, the old root weak reference is null. + } while (!roots.replace(pos, oldRef, newRef)); // If this cas fails, then try again. (Some other thread beat us to it.) + return null; // If we get here, then we successfully replaced the old root weak reference with the new one, so return null. + } + public boolean compareNullAndSet(DhLodPos pos, Node newRoot) { + WeakReference> newRef = new WeakReference<>(newRoot); + WeakReference> oldRef; + do { + oldRef = roots.putIfAbsent(pos, newRef); + if (oldRef == null) return true; // putIfAbsent succeeded + Node oldRoot = oldRef.get(); + if (oldRoot != null) return false; // putIfAbsent failed with old root not null + // Otherwise, the old root weak reference is null. + } while (!roots.replace(pos, oldRef, newRef)); // If this cas fails, then try again. (Some other thread beat us to it.) + return true; // If we get here, then we successfully replaced the old root weak reference with the new one. + } + public Node setIfNullAndGet(DhLodPos pos, Node newRoot) { + WeakReference> newRef = new WeakReference<>(newRoot); + WeakReference> oldRef; + do { + oldRef = roots.putIfAbsent(pos, newRef); + if (oldRef == null) return newRoot; // putIfAbsent succeeded + Node oldRoot = oldRef.get(); + if (oldRoot != null) return oldRoot; // putIfAbsent failed with old root not null + // Otherwise, the old root weak reference is null. + } while (!roots.replace(pos, oldRef, newRef)); // If this cas fails, then try again. (Some other thread beat us to it.) + return newRoot; // If we get here, then we successfully replaced the old root weak reference with the new one. + } + public void clean() { + roots.forEach((k,v) -> { + if (v.get() == null) // Remove the entry if the root is null + roots.remove(k, v); // But only if what we check is what we will be removing. (A CAS operation) + // Otherwise, continue. + // (It is not important that we must remove the entry if the root is null, + // as this is just a cleanup op to shrink the map.) + }); + } + } + + private final ReentrantReadWriteLock rootMapGlobalLock = new ReentrantReadWriteLock(); + private final AtomicReference> rootMap = new AtomicReference<>(new RootMap<>(0)); + + + public ConcurrentQuadCombinableProviderTree() {} + @Override + public String toString() { + return "CQCPT@" + rootMap.get().topLevel + "(~" + rootMap.get().roots.size() + ")"; + } + + // Atomically update and get the generation future + private CompletableFuture checkAndMakeFuture(Node node, Function> allNullCompleter) { + CompletableFuture future = new CompletableFuture<>(); + CompletableFuture casValue = Atomics.compareAndExchange(node.future, null, future); + if (casValue != null) { // cas failed. Existing future. Return it. + return future; + } + + // Next, we need to make the future completable. + // We first check for each child connection if it exists. If it does, we store it for a later 'allOf'. + boolean allNull = true; + @SuppressWarnings("unchecked") + CompletableFuture[] childFutures = new CompletableFuture[4]; + for (int i = 0; i < 4; i++) { + WeakReference> childRef = node.children.get(i); + Node nextChild = childRef == null ? null : childRef.get(); + if (nextChild != null) { // child node exists. Recursively make or get the child's future. + allNull = false; + childFutures[i] = checkAndMakeFuture(nextChild, allNullCompleter); + } + } + if (allNull) { // all children are null. We can then just run the allNullCompleter in this node. + allNullCompleter.apply(node.pos).thenAccept(future::complete); + } else { // some children exist. We need to wait for some or all of them to complete. + // But before that, we need to create the children node where they are missing. + for (int i = 0; i < 4; i++) { + if (childFutures[i] == null) { + CompletableFuture newChildFuture = new CompletableFuture<>(); + Node newChild = new Node<>(node.pos.getChild(i), newChildFuture, node); + node.children.set(i, new WeakReference<>(newChild)); + // Since the child is new, we can be sure that it doesn't have any children. + // So, we need to make the new child's future completable by running the allNullCompleter. + // (The above relies on the fact that we did a CAS on the beginning of this method, + // which means that we have unique access to the node and its links to the children, and that + // no other thread can be concurrently modifying its links) + allNullCompleter.apply(node.pos.getChild(i)).whenComplete((r, e) -> { + // NOTE(*1): This *HAVE* to get the future via the node reference instead of directly capturing the future, + // as otherwise the node will be garbage collected before the future is completed. + // With this, we can guarantee that the node is garbage collected only when the future is (being) completed. + // (The actual order is not important however as long as the node is still alive when the generation is in progress) + CompletableFuture f = node.future.get(); + LodUtil.assertTrue(f != null, "Future should not be null"); + if (e != null) { + f.completeExceptionally(e); + } else { + f.complete(r); + } + }); + } + } + // Now, we can wait for all the child futures to complete, and then complete this node's future with + // the combined result of all child futures. + CompletableFuture.allOf(childFutures).handle((v, e) -> { + // NOTE: Same as 'NOTE(*1)', we *HAVE* to get the future via the node reference instead of directly capturing the future. + CompletableFuture f = node.future.get(); + LodUtil.assertTrue(f != null, "Future should not be null"); + if (e != null) { + f.completeExceptionally(e); + } else { + try { + f.complete(childFutures[0].join().combineWith( + childFutures[1].join(), childFutures[2].join(), childFutures[3].join())); + } catch (Throwable e2) { + f.completeExceptionally(e2); + } + } + return null; + }); + } + return future; + } + + public CompletableFuture createOrUseExisting(DhLodPos pos, Function> completer) { + int cleanRng = ThreadLocalRandom.current().nextInt(0, 10); + if (cleanRng == 0) cleanIfNeeded(); + // First, ensure that the root map is locked for reading. (The lock is for the structure of the map, not the values) + rootMapGlobalLock.readLock().lock(); + RootMap map = rootMap.get(); + // Next, do different thing depending on the top level of the map compared to the target position. + if (map.topLevel == pos.detail) { // The target position is at the top level, meaning that we can directly use the root. + // Make the future and node first for the later CAS on null. + CompletableFuture future = new CompletableFuture<>(); + Node newNode = new Node(pos, future); // No parent node as it's the root. + Node cas = map.compareNullAndExchange(pos, newNode); // CAS the node into the map. + rootMapGlobalLock.readLock().unlock(); // We're done with the map, as following code no longer accesses it. + + if (cas == null) { // cas succeeded. Which means no existing overlapping node in same detail level. + // Reason: Since any lower level nodes should have upper level nodes as parent up to the top level, + // and that there are no same level nodes, we can assume that the new node does not overlap any existing nodes. + // Therefore, we can apply the completer function to the new node, and return the future. + completer.apply(pos).whenComplete((r, e) -> { + // See NOTE(*1) above. + CompletableFuture f = newNode.future.get(); + LodUtil.assertTrue(f != null, "Future should not be null"); + if (e != null) { + f.completeExceptionally(e); + } else { + f.complete(r); + } + }); + return future; + } else { // cas failed. Existing overlapping node. + // Run the checkAndMakeFuture method on the existing node to update and get the generation future. + return checkAndMakeFuture(cas, completer); + } + } else if (map.topLevel > pos.detail) { + // We need to traverse down the tree with the following rules during the traversal: + // 1. If the next node is not null and has a future, halt and return that future. + // 2. If the next node is not null with no future, continue traversing down the tree. + // 3. if the next node is null, create a new node and CompareExchange it into the current node, and run rule 1/2. + // Note that DO NOT assume that all subsequent nodes will fall into case 3, as someone else can concurrently + // use and modify the newly created node! + + // To start, just treat the rootMap as the... well, root, and it's content as the children node. + // We can then traverse down the tree until we reach the target node or hit the 1st case and return prematurely. + + // First iteration: + Node currentNode; + Node childNode = map.setIfNullAndGet( // rule 3: if null, create a new node. + pos.convertUpwardsTo((byte) map.topLevel), + new Node(pos, null)); // No parent node as it's the root. + rootMapGlobalLock.readLock().unlock(); // We're done with the map, as following code no longer accesses it. + + CompletableFuture future = childNode.future.get(); + if (future != null) { // rule 1: if future is not null, halt and return the future. + return future; + } else { // rule 2: if future is null, continue traversing down the tree. + currentNode = childNode; + + // Second and subsequent iterations: + while (currentNode.pos.detail > pos.detail) { + Node newNode = new Node(pos.convertUpwardsTo((byte) (currentNode.pos.detail - 1)), null, currentNode); + // Note: It is important that child link is set and created before we check the child future, + // so to avoid race conditions with checkAndMakeFuture. + childNode = currentNode.setIfNullAndGet(newNode.pos.getChildIndexOfParent(), newNode); // rule 3: if null, create a new node. + CompletableFuture childFuture = childNode.future.get(); + if (childFuture != null) { // rule 1: if future is not null, halt and return the future. + return childFuture; + } else { // rule 2: if future is null, continue traversing down the tree. + currentNode = childNode; + } + } + } + // At this point, we have reached the target node. + LodUtil.assertTrue(currentNode.pos.equals(pos)); + // We can now run the checkAndMakeFuture method on the target node to update and get the generation future. + return checkAndMakeFuture(currentNode, completer); // Technically, this will rerun the 1st rule. But code is cleaner this way. + } else { // map.topLevel < pos.detail + // Now, this is the complex case. We need to rebase the tree to the higher detail level. + // For now, this implementation will do a lock based version. However, I will figure out a way to do this without a lock. + + rootMapGlobalLock.readLock().unlock(); + while (map.topLevel < pos.detail) { + map = rebaseUpward(pos.detail); + } + LodUtil.assertTrue(map.topLevel >= pos.detail); + return createOrUseExisting(pos, completer); // After rebasing, we can just call the createOrUseExisting method again. + } + } + + private RootMap rebaseUpward(int targetLevel) { + rootMapGlobalLock.writeLock().lock(); + RootMap map = rootMap.get(); + if (map.topLevel >= targetLevel) { + rootMapGlobalLock.writeLock().unlock(); + return map; + } + // At this point, we have exclusive access to the rootMap. + map.clean(); // Clean the map. (Could actually be done with just readLock.) + RootMap newMap = new RootMap<>(map.topLevel + 1); + if (map.roots.isEmpty()) return newMap; // If there are no roots, just return the new map. + map.roots.forEach((pos, nodeRef) -> { + Node node = nodeRef.get(); + if (node == null) return; // If null, ignore that node. + LodUtil.assertTrue(pos.detail+1 == newMap.topLevel); + LodUtil.assertTrue(node.parent.get() == null); + LodUtil.assertTrue(node.pos.equals(pos)); + DhLodPos newPos = pos.convertUpwardsTo((byte) (pos.detail+1)); + + // Create the parent node, or if it already exists, use it to set the child node's parent. + // NOTE: While this section is protected by the rootMapGlobalLock, we still need to use the normal + // CAS methods to setAndGet the parent node, as the parent node may be GC'd concurrently by other threads + // who have just completed the node's future, and caused the GC parent chain up to the new map. + Node newParentNode = newMap.setIfNullAndGet(newPos, new Node(newPos, null)); + node.parent.set(newParentNode); + }); + boolean casWorked = rootMap.compareAndSet(map, newMap); + LodUtil.assertTrue(casWorked); + rootMapGlobalLock.writeLock().unlock(); + return newMap; + } + + public void cleanIfNeeded() { + if (rootMapGlobalLock.readLock().tryLock()) { + rootMap.get().clean(); + rootMapGlobalLock.readLock().unlock(); + } + } +}