From 007e99314856bca726f4d6fe6af34d6be630d07f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Fri, 25 Nov 2022 15:58:53 -0600 Subject: [PATCH] reformat chunk generator code --- .../lod/core/generation/BatchGenerator.java | 101 +- .../lod/core/generation/GenerationQueue.java | 1007 ++++++++++------- .../lod/core/generation/IChunkGenerator.java | 30 +- .../lod/core/generation/IGenerator.java | 48 +- 4 files changed, 672 insertions(+), 514 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java b/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java index 2a3850990..157f0ab4b 100644 --- a/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java +++ b/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java @@ -3,7 +3,7 @@ * licensed under the GNU LGPL v3 License. * * Copyright (C) 2021 Tom Lee (TomTheFurry) & James Seibel (Original code) - * + * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, version 3. @@ -25,6 +25,7 @@ import com.seibel.lod.api.enums.config.EDistanceGenerationMode; import com.seibel.lod.core.dependencyInjection.SingletonInjector; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.pos.DhChunkPos; +import com.seibel.lod.core.util.BitShiftUtil; import com.seibel.lod.core.wrapperInterfaces.IWrapperFactory; import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.lod.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; @@ -36,38 +37,51 @@ import java.lang.invoke.MethodHandles; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; +/** + * + * @version 2022-11-25 + */ public class BatchGenerator implements IChunkGenerator { public static final boolean ENABLE_GENERATOR_STATS_LOGGING = false; - + private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); private static final IWrapperFactory FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class); public AbstractBatchGenerationEnvionmentWrapper generationGroup; public IDhLevel targetLodLevel; public static final int generationGroupSize = 4; private static final Logger LOGGER = DhLoggerBuilder.getLogger(MethodHandles.lookup().lookupClass().getSimpleName()); - - public BatchGenerator(IDhLevel targetLodLevel) { + + + + public BatchGenerator(IDhLevel targetLodLevel) + { this.targetLodLevel = targetLodLevel; - generationGroup = FACTORY.createBatchGenerator(targetLodLevel); + this.generationGroup = FACTORY.createBatchGenerator(targetLodLevel); LOGGER.info("Batch Chunk Generator initialized"); } - - public void stop(boolean blocking) { - LOGGER.info("1.18 Experimental Chunk Generator shutting down..."); - generationGroup.stop(blocking); + + + + public void stop(boolean blocking) + { + LOGGER.info("Batch Chunk Generator shutting down..."); + this.generationGroup.stop(blocking); } - + @Override - public boolean isBusy() { - return generationGroup.getEventCount() > Math.max(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get().intValue(), 1) *1.5; + public boolean isBusy() + { + return this.generationGroup.getEventCount() > Math.max(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get().intValue(), 1) * 1.5; } - + @Override - public CompletableFuture generateChunks(DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer) { + public CompletableFuture generateChunks(DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer) + { EDistanceGenerationMode mode = Config.Client.WorldGenerator.distanceGenerationMode.get(); Steps targetStep = null; - switch (mode) { + switch (mode) + { case NONE: targetStep = Steps.Empty; // NOTE: Only load in existing chunks. No new chunk generation break; @@ -84,47 +98,36 @@ public class BatchGenerator implements IChunkGenerator case FULL: targetStep = Steps.Features; break; - }; - + } + ; + int chunkXMin = chunkPosMin.x; int chunkZMin = chunkPosMin.z; - int genChunkSize = 1 << (granularity - 4); // minus 4 for chunk size as its equal to div by 16 - double runTimeRatio = Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get()>1 ? 1.0 - : Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get(); - return generationGroup.generateChunks(chunkXMin, chunkZMin, genChunkSize, targetStep, runTimeRatio, resultConsumer); + int genChunkSize = BitShiftUtil.powerOfTwo(granularity - 4); // minus 4 for chunk size as its equal to dividing by 16 + double runTimeRatio = Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get() > 1 ? + 1.0 : + Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get(); + return this.generationGroup.generateChunks(chunkXMin, chunkZMin, genChunkSize, targetStep, runTimeRatio, resultConsumer); } - + @Override - public byte getMinDataDetail() { - return 0; - } - + public byte getMinDataDetail() { return 0; } + @Override - public byte getMaxDataDetail() { - return 0; - } - + public byte getMaxDataDetail() { return 0; } + @Override - public int getPriority() { - return 0; - } - + public int getPriority() { return 0; } + @Override - public byte getMinGenerationGranularity() { - return 4; - } - + public byte getMinGenerationGranularity() { return 4; } + @Override - public byte getMaxGenerationGranularity() { - return 6; - } - + public byte getMaxGenerationGranularity() { return 6; } + @Override - public void close() { - stop(true); - } - - public void update() { - generationGroup.updateAllFutures(); - } + public void close() { this.stop(true); } + + public void update() { this.generationGroup.updateAllFutures(); } + } diff --git a/core/src/main/java/com/seibel/lod/core/generation/GenerationQueue.java b/core/src/main/java/com/seibel/lod/core/generation/GenerationQueue.java index cf57342b1..c48d7f479 100644 --- a/core/src/main/java/com/seibel/lod/core/generation/GenerationQueue.java +++ b/core/src/main/java/com/seibel/lod/core/generation/GenerationQueue.java @@ -15,442 +15,573 @@ import java.util.*; import java.util.concurrent.*; import java.util.function.Consumer; -public class GenerationQueue implements Closeable { - public static final int SHUTDOWN_TIMEOUT_SEC = 10; - public static final int MAX_TASKS_PROCESSED_PER_TICK = 10000; - - /** - * Source: ... - * Description: Left-upper semi-diagonal (0-4-16-36-64) contains squared layer number (4 * layer^2). - * External if-statement defines layer and finds (pre-)result for position in corresponding row or - * column of left-upper semi-plane, and internal if-statement corrects result for mirror position. - */ - private static int gridSpiralIndexing(int X, int Y) { - int index = 0; - if(X*X >= Y*Y) { - index = 4 * X * X - X - Y; - if(X < Y) - index = index - 2 * (X - Y); - } else { - index = 4 *Y*Y -X - Y; - if(X < Y) - index = index + 2 * (X - Y); - } - - return index; - } - - private final Logger logger = DhLoggerBuilder.getLogger(); - public static abstract class GenTaskTracker { - public abstract boolean isValid(); - public abstract Consumer getConsumer(); - } - - final IGenerator generator; - static final class GenTask { - final DhLodPos pos; - final byte dataDetail; - final GenTaskTracker taskTracker; - final CompletableFuture future; - GenTask(DhLodPos pos, byte dataDetail, GenTaskTracker taskTracker, CompletableFuture future) { - this.dataDetail = dataDetail; - this.pos = pos; - this.taskTracker = taskTracker; - this.future = future; - } - } - static final class TaskGroup { - final DhLodPos pos; - byte dataDetail; - final LinkedList members = new LinkedList<>(); // Accessed by gen poller thread only - TaskGroup(DhLodPos pos, byte dataDetail) { - this.pos = pos; - this.dataDetail = dataDetail; - } - - void accept(ChunkSizedData data) { - Iterator iter = members.iterator(); - while (iter.hasNext()) { - GenTask task = iter.next(); - Consumer consumer = task.taskTracker.getConsumer(); - if (consumer == null) { - iter.remove(); - task.future.complete(false); - } else { - consumer.accept(data); - } - } - } - } - static final class InProgressTask { - final TaskGroup group; - CompletableFuture genFuture = null; - InProgressTask(TaskGroup group) { - this.group = group; - } - } - - static class SplitTask extends GenTaskTracker { - final GenTaskTracker parentTracker; - final CompletableFuture parentFuture; - boolean cachedValid = true; - SplitTask(GenTaskTracker parentTracker, CompletableFuture parentFuture) { - this.parentTracker = parentTracker; - this.parentFuture = parentFuture; - } - boolean recheckState() { - if (!cachedValid) return false; - cachedValid = parentTracker.isValid(); - if (!cachedValid) parentFuture.complete(false); - return cachedValid; - } - @Override - public boolean isValid() { - return cachedValid; - } - @Override - public Consumer getConsumer() { - return parentTracker.getConsumer(); - } - } - - private final ConcurrentLinkedQueue looseTasks = new ConcurrentLinkedQueue<>(); - // FIXME: Concurrency issue on close! - // FIXME: This is using up a TONS of time to process! - private final ConcurrentSkipListMap taskGroups = new ConcurrentSkipListMap<>( - (a, b) -> { - if (a.detailLevel != b.detailLevel) return a.detailLevel - b.detailLevel; - int aDist = a.getCenter().toPos2D().chebyshevDist(Pos2D.ZERO); - int bDist = b.getCenter().toPos2D().chebyshevDist(Pos2D.ZERO); - if (aDist != bDist) return aDist - bDist; - if (a.x != b.x) return a.x - b.x; - return a.z - b.z; - } - ); // Accessed by poller only - - private final ConcurrentHashMap inProgress = new ConcurrentHashMap<>(); - - private final byte maxGranularity; - private final byte minGranularity; - private final byte maxDataDetail; - private final byte minDataDetail; - private volatile CompletableFuture closer = null; - - public GenerationQueue(IGenerator generator) { - this.generator = generator; - maxGranularity = generator.getMaxGenerationGranularity(); - minGranularity = generator.getMinGenerationGranularity(); - maxDataDetail = generator.getMaxDataDetail(); - minDataDetail = generator.getMinDataDetail(); - if (minGranularity < 4) throw new IllegalArgumentException("DH-IGenerator: min granularity must be at least 4!"); - if (maxGranularity < minGranularity) throw new IllegalArgumentException("DH-IGenerator: max granularity smaller than min granularity!"); - } - - public CompletableFuture submitGenTask(DhLodPos pos, byte requiredDataDetail, GenTaskTracker tracker) { - if (closer != null) return CompletableFuture.completedFuture(false); - if (requiredDataDetail < minDataDetail) { - throw new UnsupportedOperationException("Current generator does not meet requiredDataDetail level"); - } - if (requiredDataDetail > maxDataDetail) requiredDataDetail = maxDataDetail; - - LodUtil.assertTrue(pos.detailLevel > requiredDataDetail+4); - byte granularity = (byte) (pos.detailLevel - requiredDataDetail); - - if (granularity > maxGranularity) { - // Too big of a chunk. We need to split it up - byte subDetail = (byte) (maxGranularity + requiredDataDetail); - int subPosCount = pos.getBlockWidth(subDetail); - DhLodPos cornerSubPos = pos.getCorner(subDetail); - CompletableFuture[] subFutures = new CompletableFuture[subPosCount*subPosCount]; - ArrayList subTasks = new ArrayList<>(subPosCount*subPosCount); - SplitTask splitTask = new SplitTask(tracker, new CompletableFuture<>()); - { - int i = 0; - for (int ox = 0; ox < subPosCount; ox++) { - for (int oz = 0; oz < subPosCount; oz++) { - CompletableFuture subFuture = new CompletableFuture<>(); - subFutures[i++] = subFuture; - subTasks.add(new GenTask(cornerSubPos.addOffset(ox, oz), requiredDataDetail, splitTask, subFuture)); - } - } - } - CompletableFuture.allOf(subFutures).whenComplete((v,ex) -> { - if (ex != null) splitTask.parentFuture.completeExceptionally(ex); - if (!splitTask.recheckState()) return; // Auto join future - for (CompletableFuture subFuture: subFutures) { - boolean successful = subFuture.join(); - if (!successful) { - splitTask.parentFuture.complete(false); - return; - } - } - splitTask.parentFuture.complete(true); - }); - looseTasks.addAll(subTasks); - if (closer != null) return CompletableFuture.completedFuture(false); - else return splitTask.parentFuture; - } else if (granularity < minGranularity) { - // Too small of a chunk. We'll just over-size the generation. - byte parentDetail = (byte) (minGranularity + requiredDataDetail); - DhLodPos parentPos = pos.convertUpwardsTo(parentDetail); - CompletableFuture future = new CompletableFuture<>(); - looseTasks.add(new GenTask(parentPos, requiredDataDetail, tracker, future)); - if (closer != null) return CompletableFuture.completedFuture(false); - else return future; - } else { - CompletableFuture future = new CompletableFuture<>(); - looseTasks.add(new GenTask(pos, requiredDataDetail, tracker, future)); - if (closer != null) return CompletableFuture.completedFuture(false); - else return future; - } - } - - private void addAndCombineGroup(TaskGroup target) { - byte granularity = (byte) (target.pos.detailLevel - target.dataDetail); - LodUtil.assertTrue(granularity <= maxGranularity && granularity >= minGranularity); - LodUtil.assertTrue(!taskGroups.containsKey(target.pos)); - - // Check and merge all those who has exactly the same dataDetail, and overlaps the position, but have lower granularity than us - if (granularity > minGranularity) { - // TODO: Optimize this check - Iterator groupIter = taskGroups.values().iterator(); - while (groupIter.hasNext()) { - TaskGroup group = groupIter.next(); - if (group.dataDetail != target.dataDetail) continue; - if (!group.pos.overlaps(target.pos)) continue; - - // We should have already ALWAYS selected the higher granularity. - LodUtil.assertTrue(group.pos.detailLevel < target.pos.detailLevel); - groupIter.remove(); // Remove and consume all from that lower granularity request - target.members.addAll(group.members); - } - } - - // Now, Check if we are the missing piece in the 4 quadrants, and if so, combine the four into a new higher granularity group - if (granularity < maxGranularity) { // Obviously, only do so if we aren't at the maxGranularity already - // Check for merging and upping the granularity - DhLodPos corePos = target.pos; - DhLodPos parentPos = corePos.convertUpwardsTo((byte) (corePos.detailLevel +1)); - int targetChildId = target.pos.getChildIndexOfParent(); - boolean allPassed = true; - for (int i = 0; i < 4; i++) { - if (i == targetChildId) continue; - TaskGroup group = taskGroups.get(parentPos.getChildByIndex(i)); - if (group == null || group.dataDetail != target.dataDetail) { - allPassed = false; - break; - } - } - if (allPassed) { - LodUtil.assertTrue(!taskGroups.containsKey(parentPos) || taskGroups.get(parentPos).dataDetail != target.dataDetail); - TaskGroup[] groups = new TaskGroup[4]; - for (int i = 0; i < 4; i++) { - if (i==targetChildId) groups[i] = target; - else groups[i] = taskGroups.remove(parentPos.getChildByIndex(i)); - LodUtil.assertTrue(groups[i] != null && groups[i].dataDetail == target.dataDetail); - } - - TaskGroup newGroup = taskGroups.get(parentPos); - if (newGroup != null) { - LodUtil.assertTrue(newGroup.dataDetail != target.dataDetail); // if it is equal, we should have been merged ages ago - if (newGroup.dataDetail < target.dataDetail) { - // We can just append us into the existing list. - for (TaskGroup g : groups) newGroup.members.addAll(g.members); - } else { - // We need to upgrade the requested dataDetail of the group. - newGroup.dataDetail = target.dataDetail; - boolean worked = taskGroups.remove(parentPos, newGroup); // Pop it off for later proper merge check - LodUtil.assertTrue(worked); - for (TaskGroup g : groups) newGroup.members.addAll(g.members); - addAndCombineGroup(newGroup); // Recursive check the new group - } - } else { - // There should not be any higher granularity to check, as otherwise we would have merged ages ago - newGroup = new TaskGroup(parentPos, target.dataDetail); - for (TaskGroup g : groups) newGroup.members.addAll(g.members); - addAndCombineGroup(newGroup); // Recursive check the new group - } - return; // We have merged. So no need to add the target group - } - } - - // Finally, we should be safe to add the target group into the list - TaskGroup v = taskGroups.put(target.pos, target); - LodUtil.assertTrue(v == null); // should never be replacing other things - } - - private void processLooseTasks() { - int taskProcessed = 0; - while (!looseTasks.isEmpty() && taskProcessed < MAX_TASKS_PROCESSED_PER_TICK) { - GenTask task = looseTasks.poll(); - taskProcessed++; - byte taskDataDetail = task.dataDetail; - byte taskGranularity = (byte) (task.pos.detailLevel - taskDataDetail); - LodUtil.assertTrue(taskGranularity >= 4 && taskGranularity >= minGranularity && taskGranularity <= maxGranularity); - - // Check existing one - TaskGroup group = taskGroups.get(task.pos); - if (group != null) { - if (group.dataDetail <= taskDataDetail) { - // We can just append us into the existing list. - group.members.add(task); - } else { - // We need to upgrade the requested dataDetail of the group. - group.dataDetail = taskDataDetail; - boolean worked = taskGroups.remove(task.pos, group); // Pop it off for later proper merge check - LodUtil.assertTrue(worked); - group.members.add(task); - addAndCombineGroup(group); - } - } else { - - // Check higher granularity one - byte granularity = taskGranularity; - boolean didAnything = false; - while (++granularity <= maxGranularity) { - group = taskGroups.get(task.pos.convertUpwardsTo((byte) (taskDataDetail + granularity))); - if (group != null && group.dataDetail == taskDataDetail) { - // We can just append to the higher granularity group one - group.members.add(task); - didAnything = true; - break; - } - } - if (!didAnything) { - group = new TaskGroup(task.pos, taskDataDetail); - group.members.add(task); - addAndCombineGroup(group); - } - } - } - if (taskProcessed != 0) { - logger.info("Processed " + taskProcessed + " loose tasks"); - } - - - } - - private void removeOutdatedGroups() { - // Remove all invalid genTasks and groups - Iterator groupIter = taskGroups.values().iterator(); - while (groupIter.hasNext()) { - TaskGroup group = groupIter.next(); - Iterator taskIter = group.members.iterator(); - while (taskIter.hasNext()) { - GenTask task = taskIter.next(); - if (!task.taskTracker.isValid()) { - taskIter.remove(); - task.future.complete(false); - } - } - if (group.members.isEmpty()) groupIter.remove(); - } - } - - private void pollAndStartNext(DhBlockPos2D targetPos) { - // Select the one with the highest data detail level and closest to the target pos - TaskGroup best = null; - long cachedDist = Long.MAX_VALUE; - int lastChebDist = Integer.MIN_VALUE; - boolean continueNextRound = true; - byte currentDetailChecking = -1; - - for (TaskGroup group : taskGroups.values()) { - if (currentDetailChecking == -1) currentDetailChecking = group.dataDetail; - LodUtil.assertTrue(currentDetailChecking == group.dataDetail); - int chebDistToOrigin = group.pos.getCenter().toPos2D().chebyshevDist(Pos2D.ZERO); - if (chebDistToOrigin > lastChebDist) { - if (!continueNextRound) break; // We have found the best one - continueNextRound = false; - lastChebDist = chebDistToOrigin; - } - long dist = group.pos.getCenter().distSquared(targetPos); - if (best != null && dist >= cachedDist) continue; - cachedDist = dist; - best = group; - continueNextRound = true; - } - - if (best != null) { - InProgressTask startedTask = new InProgressTask(best); - InProgressTask casTask = inProgress.putIfAbsent(best.pos, startedTask); - boolean worked = taskGroups.remove(best.pos, best); // Remove the selected task from the group - LodUtil.assertTrue(worked); - if (casTask != null) { - // Note: Due to concurrency reasons, even if the currently running task is compatible with selected task, - // we cannot use it, as some chunks may have already been written into. - pollAndStartNext(targetPos); // Poll next one. - TaskGroup exchange = taskGroups.put(best.pos, best); // put back the task. - LodUtil.assertTrue(exchange == null); - } else { - startTaskGroup(startedTask); - } - } - - } - - public void pollAndStartClosest(DhBlockPos2D targetPos) { - if (generator == null) throw new IllegalStateException("generator is null"); - if (generator.isBusy()) return; - removeOutdatedGroups(); - processLooseTasks(); - pollAndStartNext(targetPos); - } - - private void startTaskGroup(InProgressTask task) { - byte dataDetail = task.group.dataDetail; - DhLodPos pos = task.group.pos; - byte granularity = (byte) (pos.detailLevel - dataDetail); - LodUtil.assertTrue(granularity >= minGranularity && granularity <= maxGranularity); - LodUtil.assertTrue(dataDetail >= minDataDetail && dataDetail <= maxDataDetail); - - DhChunkPos chunkPosMin = new DhChunkPos(pos.getCorner()); - logger.info("Generating section {} with granularity {} at {}", pos, granularity, chunkPosMin); - task.genFuture = generator.generate( - chunkPosMin, granularity, dataDetail, task.group::accept); - task.genFuture.whenComplete((v, ex) -> { - if (ex != null) { - if (!UncheckedInterruptedException.isThrowableInterruption(ex)) - logger.error("Error generating data for section {}", pos, ex); - task.group.members.forEach(m -> m.future.complete(false)); - } else { - logger.info("Section generation at {} complated", pos); - task.group.members.forEach(m -> m.future.complete(true)); - } - boolean worked = inProgress.remove(pos, task); - LodUtil.assertTrue(worked); - }); - } - - public CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning) { - taskGroups.values().forEach(g -> g.members.forEach(t -> t.future.complete(false))); - taskGroups.clear(); - ArrayList> array = new ArrayList<>(inProgress.size()); - inProgress.values().forEach(runningTask -> - { - CompletableFuture genFuture = runningTask.genFuture; // Do this to prevent it getting swapped out - if (cancelCurrentGeneration) genFuture.cancel(alsoInterruptRunning); - array.add(genFuture.handle((v, ex) -> { - if (ex instanceof CompletionException) ex = ex.getCause(); - if (!UncheckedInterruptedException.isThrowableInterruption(ex)) - logger.error("Error when terminating data generation for section {}", runningTask.group.pos, ex); - return null; - })); - }); - closer = CompletableFuture.allOf(array.toArray(CompletableFuture[]::new)); //FIXME: Closer threading issues with pollAndStartClosest - looseTasks.forEach(t -> t.future.complete(false)); - looseTasks.clear(); - return closer; - } - - @Override - public void close() { - if (closer == null) startClosing(true, true); - LodUtil.assertTrue(closer != null); - try { - closer.orTimeout(SHUTDOWN_TIMEOUT_SEC, TimeUnit.SECONDS).join(); - } catch (Throwable e) { - logger.error("Failed to close generation queue: ", e); - } - } +/** + * @version 2022-11-25 + */ +public class GenerationQueue implements Closeable +{ + public static final int SHUTDOWN_TIMEOUT_SEC = 10; + public static final int MAX_TASKS_PROCESSED_PER_TICK = 10000; + + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + + /** + * Source: ... + * Description: Left-upper semi-diagonal (0-4-16-36-64) contains squared layer number (4 * layer^2). + * External if-statement defines layer and finds (pre-)result for position in corresponding row or + * column of left-upper semi-plane, and internal if-statement corrects result for mirror position. + */ + private static int gridSpiralIndexing(int X, int Y) + { + int index = 0; + if (X * X >= Y * Y) + { + index = 4 * X * X - X - Y; + if (X < Y) + index = index - 2 * (X - Y); + } + else + { + index = 4 * Y * Y - X - Y; + if (X < Y) + index = index + 2 * (X - Y); + } + + return index; + } + + public static abstract class GenTaskTracker + { + public abstract boolean isValid(); + + public abstract Consumer getConsumer(); + } + + final IGenerator generator; + + static final class GenTask + { + final DhLodPos pos; + final byte dataDetail; + final GenTaskTracker taskTracker; + final CompletableFuture future; + + GenTask(DhLodPos pos, byte dataDetail, GenTaskTracker taskTracker, CompletableFuture future) + { + this.dataDetail = dataDetail; + this.pos = pos; + this.taskTracker = taskTracker; + this.future = future; + } + } + + static final class TaskGroup + { + final DhLodPos pos; + byte dataDetail; + final LinkedList members = new LinkedList<>(); // Accessed by gen poller thread only + + TaskGroup(DhLodPos pos, byte dataDetail) + { + this.pos = pos; + this.dataDetail = dataDetail; + } + + void accept(ChunkSizedData data) + { + Iterator iter = this.members.iterator(); + while (iter.hasNext()) + { + GenTask task = iter.next(); + Consumer consumer = task.taskTracker.getConsumer(); + if (consumer == null) + { + iter.remove(); + task.future.complete(false); + } + else + { + consumer.accept(data); + } + } + } + } + + static final class InProgressTask + { + final TaskGroup group; + CompletableFuture genFuture = null; + + InProgressTask(TaskGroup group) + { + this.group = group; + } + } + + static class SplitTask extends GenTaskTracker + { + final GenTaskTracker parentTracker; + final CompletableFuture parentFuture; + boolean cachedValid = true; + + SplitTask(GenTaskTracker parentTracker, CompletableFuture parentFuture) + { + this.parentTracker = parentTracker; + this.parentFuture = parentFuture; + } + + boolean recheckState() + { + if (!this.cachedValid) + return false; + + this.cachedValid = this.parentTracker.isValid(); + if (!this.cachedValid) + this.parentFuture.complete(false); + + return this.cachedValid; + } + + @Override + public boolean isValid() { return this.cachedValid; } + + @Override + public Consumer getConsumer() { return this.parentTracker.getConsumer(); } + + } + + private final ConcurrentLinkedQueue looseTasks = new ConcurrentLinkedQueue<>(); + // FIXME: Concurrency issue on close! + // FIXME: This is using up a TONS of time to process! + private final ConcurrentSkipListMap taskGroups = new ConcurrentSkipListMap<>( + (a, b) -> { + if (a.detailLevel != b.detailLevel) + return a.detailLevel - b.detailLevel; + int aDist = a.getCenter().toPos2D().chebyshevDist(Pos2D.ZERO); + int bDist = b.getCenter().toPos2D().chebyshevDist(Pos2D.ZERO); + if (aDist != bDist) + return aDist - bDist; + if (a.x != b.x) + return a.x - b.x; + return a.z - b.z; + } + ); // Accessed by poller only + + private final ConcurrentHashMap inProgress = new ConcurrentHashMap<>(); + + private final byte maxGranularity; + private final byte minGranularity; + private final byte maxDataDetail; + private final byte minDataDetail; + private volatile CompletableFuture closer = null; + + public GenerationQueue(IGenerator generator) + { + this.generator = generator; + this.maxGranularity = generator.getMaxGenerationGranularity(); + this.minGranularity = generator.getMinGenerationGranularity(); + this.maxDataDetail = generator.getMaxDataDetail(); + this.minDataDetail = generator.getMinDataDetail(); + + if (this.minGranularity < 4) + throw new IllegalArgumentException("DH-IGenerator: min granularity must be at least 4!"); + if (this.maxGranularity < this.minGranularity) + throw new IllegalArgumentException("DH-IGenerator: max granularity smaller than min granularity!"); + } + + public CompletableFuture submitGenTask(DhLodPos pos, byte requiredDataDetail, GenTaskTracker tracker) + { + if (this.closer != null) + return CompletableFuture.completedFuture(false); + + if (requiredDataDetail < this.minDataDetail) + { + throw new UnsupportedOperationException("Current generator does not meet requiredDataDetail level"); + } + if (requiredDataDetail > this.maxDataDetail) + requiredDataDetail = this.maxDataDetail; + + LodUtil.assertTrue(pos.detailLevel > requiredDataDetail + 4); + byte granularity = (byte) (pos.detailLevel - requiredDataDetail); + + if (granularity > this.maxGranularity) + { + // Too big of a chunk. We need to split it up + byte subDetail = (byte) (this.maxGranularity + requiredDataDetail); + int subPosCount = pos.getBlockWidth(subDetail); + DhLodPos cornerSubPos = pos.getCorner(subDetail); + CompletableFuture[] subFutures = new CompletableFuture[subPosCount * subPosCount]; + ArrayList subTasks = new ArrayList<>(subPosCount * subPosCount); + SplitTask splitTask = new SplitTask(tracker, new CompletableFuture<>()); + { + int i = 0; + for (int ox = 0; ox < subPosCount; ox++) + { + for (int oz = 0; oz < subPosCount; oz++) + { + CompletableFuture subFuture = new CompletableFuture<>(); + subFutures[i++] = subFuture; + subTasks.add(new GenTask(cornerSubPos.addOffset(ox, oz), requiredDataDetail, splitTask, subFuture)); + } + } + } + CompletableFuture.allOf(subFutures).whenComplete((v, ex) -> { + if (ex != null) + splitTask.parentFuture.completeExceptionally(ex); + if (!splitTask.recheckState()) + return; // Auto join future + for (CompletableFuture subFuture : subFutures) + { + boolean successful = subFuture.join(); + if (!successful) + { + splitTask.parentFuture.complete(false); + return; + } + } + splitTask.parentFuture.complete(true); + }); + this.looseTasks.addAll(subTasks); + if (this.closer != null) + return CompletableFuture.completedFuture(false); + else + return splitTask.parentFuture; + } + else if (granularity < this.minGranularity) + { + // Too small of a chunk. We'll just over-size the generation. + byte parentDetail = (byte) (this.minGranularity + requiredDataDetail); + DhLodPos parentPos = pos.convertUpwardsTo(parentDetail); + CompletableFuture future = new CompletableFuture<>(); + this.looseTasks.add(new GenTask(parentPos, requiredDataDetail, tracker, future)); + if (this.closer != null) + return CompletableFuture.completedFuture(false); + else + return future; + } + else + { + CompletableFuture future = new CompletableFuture<>(); + this.looseTasks.add(new GenTask(pos, requiredDataDetail, tracker, future)); + if (this.closer != null) + return CompletableFuture.completedFuture(false); + else + return future; + } + } + + private void addAndCombineGroup(TaskGroup target) + { + byte granularity = (byte) (target.pos.detailLevel - target.dataDetail); + LodUtil.assertTrue(granularity <= this.maxGranularity && granularity >= this.minGranularity); + LodUtil.assertTrue(!this.taskGroups.containsKey(target.pos)); + + // Check and merge all those who has exactly the same dataDetail, and overlaps the position, but have lower granularity than us + if (granularity > this.minGranularity) + { + // TODO: Optimize this check + Iterator groupIter = this.taskGroups.values().iterator(); + while (groupIter.hasNext()) + { + TaskGroup group = groupIter.next(); + if (group.dataDetail != target.dataDetail) + continue; + if (!group.pos.overlaps(target.pos)) + continue; + + // We should have already ALWAYS selected the higher granularity. + LodUtil.assertTrue(group.pos.detailLevel < target.pos.detailLevel); + groupIter.remove(); // Remove and consume all from that lower granularity request + target.members.addAll(group.members); + } + } + + // Now, Check if we are the missing piece in the 4 quadrants, and if so, combine the four into a new higher granularity group + if (granularity < this.maxGranularity) + { // Obviously, only do so if we aren't at the maxGranularity already + // Check for merging and upping the granularity + DhLodPos corePos = target.pos; + DhLodPos parentPos = corePos.convertUpwardsTo((byte) (corePos.detailLevel + 1)); + int targetChildId = target.pos.getChildIndexOfParent(); + boolean allPassed = true; + for (int i = 0; i < 4; i++) + { + if (i == targetChildId) + continue; + TaskGroup group = this.taskGroups.get(parentPos.getChildByIndex(i)); + if (group == null || group.dataDetail != target.dataDetail) + { + allPassed = false; + break; + } + } + if (allPassed) + { + LodUtil.assertTrue(!this.taskGroups.containsKey(parentPos) || this.taskGroups.get(parentPos).dataDetail != target.dataDetail); + TaskGroup[] groups = new TaskGroup[4]; + for (int i = 0; i < 4; i++) + { + if (i == targetChildId) + groups[i] = target; + else + groups[i] = this.taskGroups.remove(parentPos.getChildByIndex(i)); + LodUtil.assertTrue(groups[i] != null && groups[i].dataDetail == target.dataDetail); + } + + TaskGroup newGroup = this.taskGroups.get(parentPos); + if (newGroup != null) + { + LodUtil.assertTrue(newGroup.dataDetail != target.dataDetail); // if it is equal, we should have been merged ages ago + if (newGroup.dataDetail < target.dataDetail) + { + // We can just append us into the existing list. + for (TaskGroup g : groups) + newGroup.members.addAll(g.members); + } + else + { + // We need to upgrade the requested dataDetail of the group. + newGroup.dataDetail = target.dataDetail; + boolean worked = this.taskGroups.remove(parentPos, newGroup); // Pop it off for later proper merge check + LodUtil.assertTrue(worked); + for (TaskGroup g : groups) + newGroup.members.addAll(g.members); + this.addAndCombineGroup(newGroup); // Recursive check the new group + } + } + else + { + // There should not be any higher granularity to check, as otherwise we would have merged ages ago + newGroup = new TaskGroup(parentPos, target.dataDetail); + for (TaskGroup g : groups) + newGroup.members.addAll(g.members); + this.addAndCombineGroup(newGroup); // Recursive check the new group + } + return; // We have merged. So no need to add the target group + } + } + + // Finally, we should be safe to add the target group into the list + TaskGroup v = this.taskGroups.put(target.pos, target); + LodUtil.assertTrue(v == null); // should never be replacing other things + } + + private void processLooseTasks() + { + int taskProcessed = 0; + while (!this.looseTasks.isEmpty() && taskProcessed < MAX_TASKS_PROCESSED_PER_TICK) + { + GenTask task = this.looseTasks.poll(); + taskProcessed++; + byte taskDataDetail = task.dataDetail; + byte taskGranularity = (byte) (task.pos.detailLevel - taskDataDetail); + LodUtil.assertTrue(taskGranularity >= 4 && taskGranularity >= this.minGranularity && taskGranularity <= this.maxGranularity); + + // Check existing one + TaskGroup group = this.taskGroups.get(task.pos); + if (group != null) + { + if (group.dataDetail <= taskDataDetail) + { + // We can just append us into the existing list. + group.members.add(task); + } + else + { + // We need to upgrade the requested dataDetail of the group. + group.dataDetail = taskDataDetail; + boolean worked = this.taskGroups.remove(task.pos, group); // Pop it off for later proper merge check + LodUtil.assertTrue(worked); + group.members.add(task); + this.addAndCombineGroup(group); + } + } + else + { + + // Check higher granularity one + byte granularity = taskGranularity; + boolean didAnything = false; + while (++granularity <= this.maxGranularity) + { + group = this.taskGroups.get(task.pos.convertUpwardsTo((byte) (taskDataDetail + granularity))); + if (group != null && group.dataDetail == taskDataDetail) + { + // We can just append to the higher granularity group one + group.members.add(task); + didAnything = true; + break; + } + } + if (!didAnything) + { + group = new TaskGroup(task.pos, taskDataDetail); + group.members.add(task); + this.addAndCombineGroup(group); + } + } + } + + if (taskProcessed != 0) + { + LOGGER.info("Processed " + taskProcessed + " loose tasks"); + } + + } + + private void removeOutdatedGroups() + { + // Remove all invalid genTasks and groups + Iterator groupIter = this.taskGroups.values().iterator(); + while (groupIter.hasNext()) + { + TaskGroup group = groupIter.next(); + Iterator taskIter = group.members.iterator(); + while (taskIter.hasNext()) + { + GenTask task = taskIter.next(); + if (!task.taskTracker.isValid()) + { + taskIter.remove(); + task.future.complete(false); + } + } + + if (group.members.isEmpty()) + groupIter.remove(); + } + } + + private void pollAndStartNext(DhBlockPos2D targetPos) + { + // Select the one with the highest data detail level and closest to the target pos + TaskGroup best = null; + long cachedDist = Long.MAX_VALUE; + int lastChebDist = Integer.MIN_VALUE; + boolean continueNextRound = true; + byte currentDetailChecking = -1; + + for (TaskGroup group : this.taskGroups.values()) + { + if (currentDetailChecking == -1) + currentDetailChecking = group.dataDetail; + LodUtil.assertTrue(currentDetailChecking == group.dataDetail); + int chebDistToOrigin = group.pos.getCenter().toPos2D().chebyshevDist(Pos2D.ZERO); + if (chebDistToOrigin > lastChebDist) + { + if (!continueNextRound) + break; // We have found the best one + continueNextRound = false; + lastChebDist = chebDistToOrigin; + } + long dist = group.pos.getCenter().distSquared(targetPos); + if (best != null && dist >= cachedDist) + continue; + cachedDist = dist; + best = group; + continueNextRound = true; + } + + if (best != null) + { + InProgressTask startedTask = new InProgressTask(best); + InProgressTask casTask = this.inProgress.putIfAbsent(best.pos, startedTask); + boolean worked = this.taskGroups.remove(best.pos, best); // Remove the selected task from the group + LodUtil.assertTrue(worked); + if (casTask != null) + { + // Note: Due to concurrency reasons, even if the currently running task is compatible with selected task, + // we cannot use it, as some chunks may have already been written into. + this.pollAndStartNext(targetPos); // Poll next one. + TaskGroup exchange = this.taskGroups.put(best.pos, best); // put back the task. + LodUtil.assertTrue(exchange == null); + } + else + { + this.startTaskGroup(startedTask); + } + } + + } + + public void pollAndStartClosest(DhBlockPos2D targetPos) + { + if (this.generator == null) + throw new IllegalStateException("generator is null"); + if (this.generator.isBusy()) + return; + this.removeOutdatedGroups(); + this.processLooseTasks(); + this.pollAndStartNext(targetPos); + } + + private void startTaskGroup(InProgressTask task) + { + byte dataDetail = task.group.dataDetail; + DhLodPos pos = task.group.pos; + byte granularity = (byte) (pos.detailLevel - dataDetail); + LodUtil.assertTrue(granularity >= this.minGranularity && granularity <= this.maxGranularity); + LodUtil.assertTrue(dataDetail >= this.minDataDetail && dataDetail <= this.maxDataDetail); + + DhChunkPos chunkPosMin = new DhChunkPos(pos.getCorner()); + LOGGER.info("Generating section {} with granularity {} at {}", pos, granularity, chunkPosMin); + task.genFuture = this.generator.generate( + chunkPosMin, granularity, dataDetail, task.group::accept); + task.genFuture.whenComplete((v, ex) -> { + if (ex != null) + { + if (!UncheckedInterruptedException.isThrowableInterruption(ex)) + LOGGER.error("Error generating data for section {}", pos, ex); + task.group.members.forEach(m -> m.future.complete(false)); + } + else + { + LOGGER.info("Section generation at {} complated", pos); + task.group.members.forEach(m -> m.future.complete(true)); + } + boolean worked = inProgress.remove(pos, task); + LodUtil.assertTrue(worked); + }); + } + + public CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning) + { + this.taskGroups.values().forEach(g -> g.members.forEach(t -> t.future.complete(false))); + this.taskGroups.clear(); + ArrayList> array = new ArrayList<>(inProgress.size()); + this.inProgress.values().forEach(runningTask -> + { + CompletableFuture genFuture = runningTask.genFuture; // Do this to prevent it getting swapped out + if (cancelCurrentGeneration) + genFuture.cancel(alsoInterruptRunning); + array.add(genFuture.handle((v, ex) -> { + if (ex instanceof CompletionException) + ex = ex.getCause(); + if (!UncheckedInterruptedException.isThrowableInterruption(ex)) + LOGGER.error("Error when terminating data generation for section {}", runningTask.group.pos, ex); + return null; + })); + }); + this.closer = CompletableFuture.allOf(array.toArray(CompletableFuture[]::new)); //FIXME: Closer threading issues with pollAndStartClosest + this.looseTasks.forEach(t -> t.future.complete(false)); + this.looseTasks.clear(); + return this.closer; + } + + @Override + public void close() + { + if (this.closer == null) + this.startClosing(true, true); + LodUtil.assertTrue(this.closer != null); + try + { + this.closer.orTimeout(SHUTDOWN_TIMEOUT_SEC, TimeUnit.SECONDS).join(); + } + catch (Throwable e) + { + LOGGER.error("Failed to close generation queue: ", e); + } + } } diff --git a/core/src/main/java/com/seibel/lod/core/generation/IChunkGenerator.java b/core/src/main/java/com/seibel/lod/core/generation/IChunkGenerator.java index af98fc9a6..5c9fb525d 100644 --- a/core/src/main/java/com/seibel/lod/core/generation/IChunkGenerator.java +++ b/core/src/main/java/com/seibel/lod/core/generation/IChunkGenerator.java @@ -8,14 +8,24 @@ import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; -public interface IChunkGenerator extends IGenerator { - CompletableFuture generateChunks(DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer); - - @Override - default CompletableFuture generate(DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer) { - return generateChunks(chunkPosMin, granularity, targetDataDetail, (chunk) -> { - resultConsumer.accept(LodDataBuilder.createChunkData(chunk)); - }); - } - +/** + * @version 2022-11-25 + */ +public interface IChunkGenerator extends IGenerator +{ + CompletableFuture generateChunks(DhChunkPos chunkPosMin, + byte granularity, byte targetDataDetail, + Consumer resultConsumer); + + @Override + default CompletableFuture generate(DhChunkPos chunkPosMin, + byte granularity, byte targetDataDetail, + Consumer resultConsumer) + { + return this.generateChunks(chunkPosMin, granularity, targetDataDetail, (chunk) -> + { + resultConsumer.accept(LodDataBuilder.createChunkData(chunk)); + }); + } + } diff --git a/core/src/main/java/com/seibel/lod/core/generation/IGenerator.java b/core/src/main/java/com/seibel/lod/core/generation/IGenerator.java index ed5f7c0b7..b72506b25 100644 --- a/core/src/main/java/com/seibel/lod/core/generation/IGenerator.java +++ b/core/src/main/java/com/seibel/lod/core/generation/IGenerator.java @@ -6,29 +6,43 @@ import com.seibel.lod.core.pos.DhChunkPos; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; -public interface IGenerator extends AutoCloseable { - // What is the detail / resolution of the data? (This will offset the generation granularity) - // (minimum detail is 0, maximum detail is 255) (though that high isn't really... realistic) - // (0 = 1x1 block per data, 1 = 2x2 block per data, 2 = 4x4 block per data... etc.) - // TODO: System currently only supports 1x1 block per data. +/** + * @version 2022-11-25 + */ +public interface IGenerator extends AutoCloseable +{ + /** + * What is the detail/resolution of the data? (This will offset the generation granularity) + * (minimum detail is 0, maximum detail is 255) (though that high isn't really... realistic) + * (0 = 1x1 block per data, 1 = 2x2 block per data, 2 = 4x4 block per data... etc.) + * TODO: System currently only supports 1x1 block per data. + */ byte getMinDataDetail(); byte getMaxDataDetail(); int getPriority(); - // What is the min batch size of a single generation? - // (minimum return value is 4 since that's the MC chunk size) - // (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.) + + /** + * What is the min batch size of a single generation? + * (minimum return value is 4 since that's the MC chunk size) + * (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.) + */ byte getMinGenerationGranularity(); - - // What is the max batch size of a single generation? The system will try to group tasks to the max batch size if possible - // (minimum return value is 4 since that's the MC chunk size) - // (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.) + + /** + * What is the max batch size of a single generation? The system will try to group tasks to the max batch size if possible + * (minimum return value is 4 since that's the MC chunk size) + * (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.) + */ byte getMaxGenerationGranularity(); - - // Start a generation event - // (Note that the chunkPos is always aligned to the granularity) - // (For example, if the granularity is 4, data detail is 0, the chunkPos will be aligned to 16x16 blocks) + + /** + * Start a generation event + * (Note that the chunkPos is always aligned to the granularity) + * (For example, if the granularity is 4, data detail is 0, the chunkPos will be aligned to 16x16 blocks) + */ CompletableFuture generate(DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer); - // Return whether the generator is currently busy and cannot accept new generation requests. + /** Returns whether the generator is unable to accept new generation requests. */ boolean isBusy(); + }