From 8f6109768cbf0ebe07234cf774bf06fa817d7f99 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 5 Jun 2023 19:50:21 -0500 Subject: [PATCH] Move the world gen thread pool into the WorldGenQueue We want Core to handle the world gen threads, not the individual world generators. --- .../override/AbstractDhApiWorldGenerator.java | 31 ------ .../worldGenerator/IDhApiWorldGenerator.java | 5 +- .../lod/core/api/internal/SharedApi.java | 19 ++-- .../com/seibel/lod/core/config/Config.java | 14 +-- .../lod/core/generation/BatchGenerator.java | 26 +++-- .../core/generation/WorldGenerationQueue.java | 105 +++++++++++++----- ...ractBatchGenerationEnvironmentWrapper.java | 9 +- .../objects/TestWorldGenerator.java | 3 +- 8 files changed, 119 insertions(+), 93 deletions(-) delete mode 100644 api/src/main/java/com/seibel/lod/api/interfaces/override/AbstractDhApiWorldGenerator.java diff --git a/api/src/main/java/com/seibel/lod/api/interfaces/override/AbstractDhApiWorldGenerator.java b/api/src/main/java/com/seibel/lod/api/interfaces/override/AbstractDhApiWorldGenerator.java deleted file mode 100644 index 9d84b0833..000000000 --- a/api/src/main/java/com/seibel/lod/api/interfaces/override/AbstractDhApiWorldGenerator.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.seibel.lod.api.interfaces.override; - -import com.seibel.lod.api.enums.worldGeneration.EDhApiDistantGeneratorMode; -import com.seibel.lod.api.interfaces.world.IDhApiLevelWrapper; -import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenThreadMode; -import com.seibel.lod.api.interfaces.world.IDhApiChunkWrapper; - -/** - * @author James Seibel - * @version 2022-9-8 - */ -public abstract class AbstractDhApiWorldGenerator implements IDhApiOverrideable -{ - /** Returns which thread chunk generation requests can be created on. */ - public abstract EDhApiWorldGenThreadMode getThreadingMode(); - - public EDhApiWorldGenThreadMode getCoreThreadingMode() - { - return this.getThreadingMode(); - } - - public abstract IDhApiChunkWrapper generateChunk(int chunkPosX, int chunkPosZ, IDhApiLevelWrapper serverLevelWrapper, EDhApiDistantGeneratorMode maxStepToGenerate); - - public final IDhApiChunkWrapper generateCoreChunk(int chunkPosX, int chunkPosZ, IDhApiLevelWrapper serverLevelWrapper, EDhApiDistantGeneratorMode maxStepToGenerate) - { - // TODO probably need to change the return type - return null; //generateChunk(chunkPosX, chunkPosZ, null, generationStepEnumConverter.convertToApiType(maxStepToGenerate)); - } - - -} diff --git a/api/src/main/java/com/seibel/lod/api/interfaces/override/worldGenerator/IDhApiWorldGenerator.java b/api/src/main/java/com/seibel/lod/api/interfaces/override/worldGenerator/IDhApiWorldGenerator.java index bc4187d04..4ab8f8b27 100644 --- a/api/src/main/java/com/seibel/lod/api/interfaces/override/worldGenerator/IDhApiWorldGenerator.java +++ b/api/src/main/java/com/seibel/lod/api/interfaces/override/worldGenerator/IDhApiWorldGenerator.java @@ -7,11 +7,12 @@ import com.seibel.lod.api.interfaces.override.IDhApiOverrideable; import java.io.Closeable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; import java.util.function.Consumer; /** * @author James Seibel - * @version 2022-12-10 + * @version 2023-6-5 */ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable { @@ -97,7 +98,7 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable */ CompletableFuture generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode, - Consumer resultConsumer) throws ClassCastException; + ExecutorService worldGeneratorThreadPool, Consumer resultConsumer) throws ClassCastException; diff --git a/core/src/main/java/com/seibel/lod/core/api/internal/SharedApi.java b/core/src/main/java/com/seibel/lod/core/api/internal/SharedApi.java index 78da9dfce..a080e107b 100644 --- a/core/src/main/java/com/seibel/lod/core/api/internal/SharedApi.java +++ b/core/src/main/java/com/seibel/lod/core/api/internal/SharedApi.java @@ -4,6 +4,7 @@ import com.seibel.lod.core.Initializer; import com.seibel.lod.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; import com.seibel.lod.core.dataObjects.transformers.DataRenderTransformer; import com.seibel.lod.core.file.fullDatafile.FullDataFileHandler; +import com.seibel.lod.core.generation.WorldGenerationQueue; import com.seibel.lod.core.world.*; import com.seibel.lod.core.wrapperInterfaces.minecraft.IMinecraftSharedWrapper; @@ -28,17 +29,21 @@ public class SharedApi // starting and stopping the DataRenderTransformer is necessary to prevent attempting to // access the MC level at inappropriate times, which can cause exceptions - if (currentWorld == null) - { - DataRenderTransformer.shutdownExecutorService(); - FullDataFileHandler.shutdownExecutorService(); - ColumnRenderBufferBuilder.shutdownExecutorService(); - } - else + if (currentWorld != null) { + // thread pool setup DataRenderTransformer.setupExecutorService(); FullDataFileHandler.setupExecutorService(); ColumnRenderBufferBuilder.setupExecutorService(); + WorldGenerationQueue.setupWorldGenThreadPool(); + } + else + { + // thread pool shutdown + DataRenderTransformer.shutdownExecutorService(); + FullDataFileHandler.shutdownExecutorService(); + ColumnRenderBufferBuilder.shutdownExecutorService(); + WorldGenerationQueue.shutdownWorldGenThreadPool(); } } diff --git a/core/src/main/java/com/seibel/lod/core/config/Config.java b/core/src/main/java/com/seibel/lod/core/config/Config.java index 6a6b639c7..1452d2874 100644 --- a/core/src/main/java/com/seibel/lod/core/config/Config.java +++ b/core/src/main/java/com/seibel/lod/core/config/Config.java @@ -703,19 +703,17 @@ public class Config + " This can be an issue when first loading into a world, when flying, and/or when generating new terrain."; - public static final ConfigEntry numberOfWorldGenerationThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.1, - (double) Runtime.getRuntime().availableProcessors()/6, - (double) Runtime.getRuntime().availableProcessors()) + public static final ConfigEntry numberOfWorldGenerationThreads = new ConfigEntry.Builder() + .setMinDefaultMax(1, + Runtime.getRuntime().availableProcessors()/6, + Runtime.getRuntime().availableProcessors()) .comment("" + " How many threads should be used when generating LOD \n" + " chunks outside the normal render distance? \n" + "\n" - + " If it's less than 1, it will be treated as a percentage \n" - + " of time a single thread can run before going to idle. \n" - + "\n" + " If you experience stuttering when generating distant LODs, \n" - + " decrease this number. If you want to increase LOD \n" + + " decrease this number. \n" + + " If you want to increase LOD \n" + " generation speed, increase this number. \n" + "\n" + THREAD_NOTE) diff --git a/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java b/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java index f19552bd5..548130f18 100644 --- a/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java +++ b/core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java @@ -36,6 +36,7 @@ import com.seibel.lod.core.wrapperInterfaces.worldGeneration.AbstractBatchGenera import org.apache.logging.log4j.Logger; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; import java.util.function.Consumer; /** @@ -56,16 +57,19 @@ public class BatchGenerator implements IDhApiWorldGenerator */ private static final int MAX_QUEUED_TASKS = 3; - public AbstractBatchGenerationEnvironmentWrapper generationGroup; + public AbstractBatchGenerationEnvironmentWrapper generationEnvironment; public IDhLevel targetDhLevel; + //=============// + // constructor // + //=============// public BatchGenerator(IDhLevel targetDhLevel) { this.targetDhLevel = targetDhLevel; - this.generationGroup = FACTORY.createBatchGenerator(targetDhLevel); + this.generationEnvironment = FACTORY.createBatchGenerator(targetDhLevel); LOGGER.info("Batch Chunk Generator initialized"); } @@ -105,7 +109,9 @@ public class BatchGenerator implements IDhApiWorldGenerator //===================// @Override - public CompletableFuture generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode, Consumer resultConsumer) + public CompletableFuture generateChunks( + int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode, + ExecutorService worldGeneratorThreadPool, Consumer resultConsumer) { EDhApiWorldGenerationStep targetStep = null; switch (generatorMode) @@ -129,23 +135,19 @@ public class BatchGenerator implements IDhApiWorldGenerator } int genChunkSize = BitShiftUtil.powerOfTwo(granularity - 4); // minus 4 is equal to dividing by 16 to convert to chunk scale - double runTimeRatio = Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get() > 1 ? - 1.0 : - Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get(); // the consumer needs to be wrapped like this because the API can't use DH core objects (and IChunkWrapper can't be easily put into the API project) - Consumer consumer = (chunkWrapper) -> resultConsumer.accept(new Object[]{ chunkWrapper }); - - return this.generationGroup.generateChunks(chunkPosMinX, chunkPosMinZ, genChunkSize, targetStep, runTimeRatio, consumer); + Consumer consumerWrapper = (chunkWrapper) -> resultConsumer.accept(new Object[]{ chunkWrapper }); + return this.generationEnvironment.generateChunks(chunkPosMinX, chunkPosMinZ, genChunkSize, targetStep, worldGeneratorThreadPool, consumerWrapper); } @Override - public void preGeneratorTaskStart() { this.generationGroup.updateAllFutures(); } + public void preGeneratorTaskStart() { this.generationEnvironment.updateAllFutures(); } @Override public boolean isBusy() { - return this.generationGroup.getEventCount() > Math.max(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get().intValue(), 1) * MAX_QUEUED_TASKS; + return this.generationEnvironment.getEventCount() > Math.max(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get().intValue(), 1) * MAX_QUEUED_TASKS; } @@ -158,7 +160,7 @@ public class BatchGenerator implements IDhApiWorldGenerator public void close() { LOGGER.info(BatchGenerator.class.getSimpleName()+" shutting down..."); - this.generationGroup.stop(true); + this.generationEnvironment.stop(); } diff --git a/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java index a445b847a..e3d9a7a10 100644 --- a/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java @@ -3,10 +3,11 @@ package com.seibel.lod.core.generation; import com.seibel.lod.api.enums.worldGeneration.EDhApiDistantGeneratorMode; import com.seibel.lod.api.interfaces.override.worldGenerator.IDhApiWorldGenerator; import com.seibel.lod.core.config.Config; +import com.seibel.lod.core.config.listeners.ConfigChangeListener; import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.lod.core.dataObjects.transformers.LodDataBuilder; import com.seibel.lod.core.dependencyInjection.SingletonInjector; -import com.seibel.lod.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.lod.core.file.fullDatafile.FullDataFileHandler; import com.seibel.lod.core.generation.tasks.*; import com.seibel.lod.core.pos.*; import com.seibel.lod.core.util.ThreadUtil; @@ -20,7 +21,6 @@ import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import org.apache.logging.log4j.Logger; import java.io.Closeable; -import java.lang.ref.WeakReference; import java.util.*; import java.util.concurrent.*; import java.util.function.Consumer; @@ -65,6 +65,9 @@ public class WorldGenerationQueue implements Closeable private final HashSet alreadyGeneratedPosHashSet = new HashSet<>(MAX_ALREADY_GENERATED_COUNT); private final Queue alreadyGeneratedPosQueue = new LinkedList<>(); + private static ExecutorService worldGeneratorThreadPool; + private static ConfigChangeListener configListener; + //==============// @@ -393,7 +396,7 @@ public class WorldGenerationQueue implements Closeable //LOGGER.info("Generating section "+taskPos+" with granularity "+granularity+" at "+chunkPosMin); this.numberOfTasksQueued++; - inProgressTaskGroup.genFuture = startGenerationEvent(this.generator, chunkPosMin, granularity, taskDetailLevel, inProgressTaskGroup.group::onGenerationComplete); + inProgressTaskGroup.genFuture = this.startGenerationEvent(chunkPosMin, granularity, taskDetailLevel, inProgressTaskGroup.group::onGenerationComplete); inProgressTaskGroup.genFuture.whenComplete((voidObj, exception) -> { this.numberOfTasksQueued--; @@ -421,22 +424,22 @@ public class WorldGenerationQueue implements Closeable * The chunkPos is always aligned to the granularity. * For example: if the granularity is 4 (chunk sized) with a data detail level of 0 (block sized), the chunkPos will be aligned to 16x16 blocks. */ - private static CompletableFuture startGenerationEvent(IDhApiWorldGenerator worldGenerator, + private CompletableFuture startGenerationEvent( DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer generationCompleteConsumer) { EDhApiDistantGeneratorMode generatorMode = Config.Client.WorldGenerator.distantGeneratorMode.get(); - return worldGenerator.generateChunks(chunkPosMin.x, chunkPosMin.z, granularity, targetDataDetail, generatorMode, (objectArray) -> + return this.generator.generateChunks(chunkPosMin.x, chunkPosMin.z, granularity, targetDataDetail, generatorMode, worldGeneratorThreadPool, (generatedObjectArray) -> { try { - IChunkWrapper chunk = SingletonInjector.INSTANCE.get(IWrapperFactory.class).createChunkWrapper(objectArray); + IChunkWrapper chunk = SingletonInjector.INSTANCE.get(IWrapperFactory.class).createChunkWrapper(generatedObjectArray); generationCompleteConsumer.accept(LodDataBuilder.createChunkData(chunk)); } catch (ClassCastException e) { - DhLoggerBuilder.getLogger().error("World generator return type incorrect. Error: [" + e.getMessage() + "].", e); + DhLoggerBuilder.getLogger().error("World generator return type incorrect. Error: ["+e.getMessage()+"]. World generator disabled.", e); Config.Client.WorldGenerator.enableDistantGeneration.set(false); } }); @@ -444,31 +447,53 @@ public class WorldGenerationQueue implements Closeable + //==========================// + // executor handler methods // + //==========================// + + /** + * Creates a new executor.
+ * Does nothing if an executor already exists. + */ + public static void setupWorldGenThreadPool() + { + // static setup + if (configListener == null) + { + configListener = new ConfigChangeListener<>(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads, (threadCount) -> { setThreadPoolSize(threadCount); }); + } + + + if (worldGeneratorThreadPool == null || worldGeneratorThreadPool.isTerminated()) + { + LOGGER.info("Starting "+ FullDataFileHandler.class.getSimpleName()); + setThreadPoolSize(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get()); + } + } + public static void setThreadPoolSize(int threadPoolSize) { worldGeneratorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "DH-Gen-Worker-Thread", Thread.MIN_PRIORITY); } + + /** + * Stops any executing tasks and destroys the executor.
+ * Does nothing if the executor isn't running. + */ + public static void shutdownWorldGenThreadPool() + { + if (worldGeneratorThreadPool != null) + { + LOGGER.info("Stopping "+ FullDataFileHandler.class.getSimpleName()); + worldGeneratorThreadPool.shutdownNow(); + } + } + + + //==========// // shutdown // //==========// public CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning) { - queueingThread.shutdownNow(); - - // remove any incomplete generation tasks -// for (byte detailLevel = QuadTree.TREE_LOWEST_DETAIL_LEVEL; detailLevel < this.waitingTaskQuadTree.treeMaxDetailLevel; detailLevel++) -// { -// MovableGridRingList ringList = this.waitingTaskQuadTree.getRingList(detailLevel); -// ringList.clear((worldGenTask) -> -// { -// if (worldGenTask != null) -// { -// try -// { -// worldGenTask.future.cancel(true); -// } -// catch (CancellationException ignored) -// { /* don't log shutdown exceptions */ } -// } -// }); -// } + this.queueingThread.shutdownNow(); // stop and remove any in progress tasks @@ -519,16 +544,40 @@ public class WorldGenerationQueue implements Closeable } LodUtil.assertTrue(this.generatorClosingFuture != null); + + + + LOGGER.info("Awaiting world generator thread pool termination..."); + try + { + int waitTimeInSeconds = 3; + if (!worldGeneratorThreadPool.awaitTermination(waitTimeInSeconds, TimeUnit.SECONDS)) + { + LOGGER.warn("World generator thread pool shutdown didn't complete after ["+waitTimeInSeconds+"] seconds. Some world generator requests may still be running."); + } + } + catch (InterruptedException e) + { + LOGGER.warn("World generator thread pool shutdown interrupted! Ignoring child threads...", e); + } + + + + this.generator.close(); + + + try { this.generatorClosingFuture.cancel(true); } catch (Throwable e) { - LOGGER.error("Failed to close generation queue: ", e); + LOGGER.warn("Failed to close generation queue: ", e); } - LOGGER.info("Successfully closed "+WorldGenerationQueue.class.getSimpleName()); + + LOGGER.info("Finished closing "+WorldGenerationQueue.class.getSimpleName()); } diff --git a/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvironmentWrapper.java b/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvironmentWrapper.java index b9c0d9556..fd9342246 100644 --- a/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvironmentWrapper.java +++ b/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvironmentWrapper.java @@ -24,20 +24,21 @@ import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; import java.util.function.Consumer; public abstract class AbstractBatchGenerationEnvironmentWrapper { public AbstractBatchGenerationEnvironmentWrapper(IDhLevel level) { } - public abstract void resizeThreadPool(int newThreadCount); - public abstract void updateAllFutures(); public abstract int getEventCount(); - public abstract void stop(boolean blocking); + public abstract void stop(); - public abstract CompletableFuture generateChunks(int minX, int minZ, int genSize, EDhApiWorldGenerationStep targetStep, double runTimeRatio, Consumer resultConsumer); + public abstract CompletableFuture generateChunks( + int minX, int minZ, int genSize, EDhApiWorldGenerationStep targetStep, + ExecutorService worldGeneratorThreadPool, Consumer resultConsumer); } diff --git a/core/src/test/java/testItems/worldGeneratorInjection/objects/TestWorldGenerator.java b/core/src/test/java/testItems/worldGeneratorInjection/objects/TestWorldGenerator.java index 436729043..63546d2cc 100644 --- a/core/src/test/java/testItems/worldGeneratorInjection/objects/TestWorldGenerator.java +++ b/core/src/test/java/testItems/worldGeneratorInjection/objects/TestWorldGenerator.java @@ -7,6 +7,7 @@ import com.seibel.lod.coreapi.DependencyInjection.OverrideInjector; import com.seibel.lod.core.util.LodUtil; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; import java.util.function.Consumer; /** @@ -61,7 +62,7 @@ public class TestWorldGenerator implements IDhApiWorldGenerator public boolean isBusy() { return false; } @Override - public CompletableFuture generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode maxGenerationStep, Consumer resultConsumer) { return null; } + public CompletableFuture generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode maxGenerationStep, ExecutorService executorService, Consumer resultConsumer) { return null; } @Override public void preGeneratorTaskStart() { }