wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move the world gen thread pool into the WorldGenQueue

Browse Source 
We want Core to handle the world gen threads, not the individual world generators.
This commit is contained in:
James Seibel
2023-06-05 19:50:21 -05:00
parent d64446ecda
commit 8f6109768c
8 changed files with 119 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/java/com/seibel/lod/api/interfaces/override/AbstractDhApiWorldGenerator.java
-31
View File
@@ -1,31 +0,0 @@
package com.seibel.lod.api.interfaces.override;
import com.seibel.lod.api.enums.worldGeneration.EDhApiDistantGeneratorMode;
import com.seibel.lod.api.interfaces.world.IDhApiLevelWrapper;
import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenThreadMode;
import com.seibel.lod.api.interfaces.world.IDhApiChunkWrapper;
/**
* @author James Seibel
* @version 2022-9-8
*/
public abstract class AbstractDhApiWorldGenerator implements IDhApiOverrideable
{
/** Returns which thread chunk generation requests can be created on. */
public abstract EDhApiWorldGenThreadMode getThreadingMode();
public EDhApiWorldGenThreadMode getCoreThreadingMode()
{
return this.getThreadingMode();
}
public abstract IDhApiChunkWrapper generateChunk(int chunkPosX, int chunkPosZ, IDhApiLevelWrapper serverLevelWrapper, EDhApiDistantGeneratorMode maxStepToGenerate);
public final IDhApiChunkWrapper generateCoreChunk(int chunkPosX, int chunkPosZ, IDhApiLevelWrapper serverLevelWrapper, EDhApiDistantGeneratorMode maxStepToGenerate)
{
// TODO probably need to change the return type
return null; //generateChunk(chunkPosX, chunkPosZ, null, generationStepEnumConverter.convertToApiType(maxStepToGenerate));
}
}
api/src/main/java/com/seibel/lod/api/interfaces/override/worldGenerator/IDhApiWorldGenerator.java
+3 -2
View File
@@ -7,11 +7,12 @@ import com.seibel.lod.api.interfaces.override.IDhApiOverrideable;
import java.io.Closeable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
/**
* @author James Seibel
* @version 2022-12-10
* @version 2023-6-5
*/
public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
{
@@ -97,7 +98,7 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
*/
CompletableFuture<Void> generateChunks(int chunkPosMinX, int chunkPosMinZ,
byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode,
Consumer<Object[]> resultConsumer) throws ClassCastException;
ExecutorService worldGeneratorThreadPool, Consumer<Object[]> resultConsumer) throws ClassCastException;
core/src/main/java/com/seibel/lod/core/api/internal/SharedApi.java
+12 -7
View File
@@ -4,6 +4,7 @@ import com.seibel.lod.core.Initializer;
import com.seibel.lod.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder;
import com.seibel.lod.core.dataObjects.transformers.DataRenderTransformer;
import com.seibel.lod.core.file.fullDatafile.FullDataFileHandler;
import com.seibel.lod.core.generation.WorldGenerationQueue;
import com.seibel.lod.core.world.*;
import com.seibel.lod.core.wrapperInterfaces.minecraft.IMinecraftSharedWrapper;
@@ -28,17 +29,21 @@ public class SharedApi
// starting and stopping the DataRenderTransformer is necessary to prevent attempting to
// access the MC level at inappropriate times, which can cause exceptions
if (currentWorld == null)
{
DataRenderTransformer.shutdownExecutorService();
FullDataFileHandler.shutdownExecutorService();
ColumnRenderBufferBuilder.shutdownExecutorService();
}
else
if (currentWorld != null)
{
// thread pool setup
DataRenderTransformer.setupExecutorService();
FullDataFileHandler.setupExecutorService();
ColumnRenderBufferBuilder.setupExecutorService();
WorldGenerationQueue.setupWorldGenThreadPool();
}
else
{
// thread pool shutdown
DataRenderTransformer.shutdownExecutorService();
FullDataFileHandler.shutdownExecutorService();
ColumnRenderBufferBuilder.shutdownExecutorService();
WorldGenerationQueue.shutdownWorldGenThreadPool();
}
}
core/src/main/java/com/seibel/lod/core/config/Config.java
+6 -8
View File
@@ -703,19 +703,17 @@ public class Config
+ " This can be an issue when first loading into a world, when flying, and/or when generating new terrain.";
public static final ConfigEntry<Double> numberOfWorldGenerationThreads = new ConfigEntry.Builder<Double>()
.setMinDefaultMax(0.1,
(double) Runtime.getRuntime().availableProcessors()/6,
(double) Runtime.getRuntime().availableProcessors())
public static final ConfigEntry<Integer> numberOfWorldGenerationThreads = new ConfigEntry.Builder<Integer>()
.setMinDefaultMax(1,
Runtime.getRuntime().availableProcessors()/6,
Runtime.getRuntime().availableProcessors())
.comment(""
+ " How many threads should be used when generating LOD \n"
+ " chunks outside the normal render distance? \n"
+ "\n"
+ " If it's less than 1, it will be treated as a percentage \n"
+ " of time a single thread can run before going to idle. \n"
+ "\n"
+ " If you experience stuttering when generating distant LODs, \n"
+ " decrease this number. If you want to increase LOD \n"
+ " decrease this number. \n"
+ " If you want to increase LOD \n"
+ " generation speed, increase this number. \n"
+ "\n"
+ THREAD_NOTE)
core/src/main/java/com/seibel/lod/core/generation/BatchGenerator.java
+14 -12
View File
@@ -36,6 +36,7 @@ import com.seibel.lod.core.wrapperInterfaces.worldGeneration.AbstractBatchGenera
import org.apache.logging.log4j.Logger;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
/**
@@ -56,16 +57,19 @@ public class BatchGenerator implements IDhApiWorldGenerator
*/
private static final int MAX_QUEUED_TASKS = 3;
public AbstractBatchGenerationEnvironmentWrapper generationGroup;
public AbstractBatchGenerationEnvironmentWrapper generationEnvironment;
public IDhLevel targetDhLevel;
//=============//
// constructor //
//=============//
public BatchGenerator(IDhLevel targetDhLevel)
{
this.targetDhLevel = targetDhLevel;
this.generationGroup = FACTORY.createBatchGenerator(targetDhLevel);
this.generationEnvironment = FACTORY.createBatchGenerator(targetDhLevel);
LOGGER.info("Batch Chunk Generator initialized");
}
@@ -105,7 +109,9 @@ public class BatchGenerator implements IDhApiWorldGenerator
//===================//
@Override
public CompletableFuture<Void> generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode, Consumer<Object[]> resultConsumer)
public CompletableFuture<Void> generateChunks(
int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode,
ExecutorService worldGeneratorThreadPool, Consumer<Object[]> resultConsumer)
{
EDhApiWorldGenerationStep targetStep = null;
switch (generatorMode)
@@ -129,23 +135,19 @@ public class BatchGenerator implements IDhApiWorldGenerator
}
int genChunkSize = BitShiftUtil.powerOfTwo(granularity - 4); // minus 4 is equal to dividing by 16 to convert to chunk scale
double runTimeRatio = Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get() > 1 ?
1.0 :
Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get();
// the consumer needs to be wrapped like this because the API can't use DH core objects (and IChunkWrapper can't be easily put into the API project)
Consumer<IChunkWrapper> consumer = (chunkWrapper) -> resultConsumer.accept(new Object[]{ chunkWrapper });
return this.generationGroup.generateChunks(chunkPosMinX, chunkPosMinZ, genChunkSize, targetStep, runTimeRatio, consumer);
Consumer<IChunkWrapper> consumerWrapper = (chunkWrapper) -> resultConsumer.accept(new Object[]{ chunkWrapper });
return this.generationEnvironment.generateChunks(chunkPosMinX, chunkPosMinZ, genChunkSize, targetStep, worldGeneratorThreadPool, consumerWrapper);
}
@Override
public void preGeneratorTaskStart() { this.generationGroup.updateAllFutures(); }
public void preGeneratorTaskStart() { this.generationEnvironment.updateAllFutures(); }
@Override
public boolean isBusy()
{
return this.generationGroup.getEventCount() > Math.max(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get().intValue(), 1) * MAX_QUEUED_TASKS;
return this.generationEnvironment.getEventCount() > Math.max(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get().intValue(), 1) * MAX_QUEUED_TASKS;
}
@@ -158,7 +160,7 @@ public class BatchGenerator implements IDhApiWorldGenerator
public void close()
{
LOGGER.info(BatchGenerator.class.getSimpleName()+" shutting down...");
this.generationGroup.stop(true);
this.generationEnvironment.stop();
}
core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java
+77 -28
View File
@@ -3,10 +3,11 @@ package com.seibel.lod.core.generation;
import com.seibel.lod.api.enums.worldGeneration.EDhApiDistantGeneratorMode;
import com.seibel.lod.api.interfaces.override.worldGenerator.IDhApiWorldGenerator;
import com.seibel.lod.core.config.Config;
import com.seibel.lod.core.config.listeners.ConfigChangeListener;
import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.lod.core.dataObjects.transformers.LodDataBuilder;
import com.seibel.lod.core.dependencyInjection.SingletonInjector;
import com.seibel.lod.core.file.fullDatafile.GeneratedFullDataFileHandler;
import com.seibel.lod.core.file.fullDatafile.FullDataFileHandler;
import com.seibel.lod.core.generation.tasks.*;
import com.seibel.lod.core.pos.*;
import com.seibel.lod.core.util.ThreadUtil;
@@ -20,7 +21,6 @@ import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper;
import org.apache.logging.log4j.Logger;
import java.io.Closeable;
import java.lang.ref.WeakReference;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.Consumer;
@@ -65,6 +65,9 @@ public class WorldGenerationQueue implements Closeable
private final HashSet<DhLodPos> alreadyGeneratedPosHashSet = new HashSet<>(MAX_ALREADY_GENERATED_COUNT);
private final Queue<DhLodPos> alreadyGeneratedPosQueue = new LinkedList<>();
private static ExecutorService worldGeneratorThreadPool;
private static ConfigChangeListener<Integer> configListener;
//==============//
@@ -393,7 +396,7 @@ public class WorldGenerationQueue implements Closeable
//LOGGER.info("Generating section "+taskPos+" with granularity "+granularity+" at "+chunkPosMin);
this.numberOfTasksQueued++;
inProgressTaskGroup.genFuture = startGenerationEvent(this.generator, chunkPosMin, granularity, taskDetailLevel, inProgressTaskGroup.group::onGenerationComplete);
inProgressTaskGroup.genFuture = this.startGenerationEvent(chunkPosMin, granularity, taskDetailLevel, inProgressTaskGroup.group::onGenerationComplete);
inProgressTaskGroup.genFuture.whenComplete((voidObj, exception) ->
{
this.numberOfTasksQueued--;
@@ -421,22 +424,22 @@ public class WorldGenerationQueue implements Closeable
* The chunkPos is always aligned to the granularity.
* For example: if the granularity is 4 (chunk sized) with a data detail level of 0 (block sized), the chunkPos will be aligned to 16x16 blocks.
*/
private static CompletableFuture<Void> startGenerationEvent(IDhApiWorldGenerator worldGenerator,
private CompletableFuture<Void> startGenerationEvent(
DhChunkPos chunkPosMin,
byte granularity, byte targetDataDetail,
Consumer<ChunkSizedFullDataAccessor> generationCompleteConsumer)
{
EDhApiDistantGeneratorMode generatorMode = Config.Client.WorldGenerator.distantGeneratorMode.get();
return worldGenerator.generateChunks(chunkPosMin.x, chunkPosMin.z, granularity, targetDataDetail, generatorMode, (objectArray) ->
return this.generator.generateChunks(chunkPosMin.x, chunkPosMin.z, granularity, targetDataDetail, generatorMode, worldGeneratorThreadPool, (generatedObjectArray) ->
{
try
{
IChunkWrapper chunk = SingletonInjector.INSTANCE.get(IWrapperFactory.class).createChunkWrapper(objectArray);
IChunkWrapper chunk = SingletonInjector.INSTANCE.get(IWrapperFactory.class).createChunkWrapper(generatedObjectArray);
generationCompleteConsumer.accept(LodDataBuilder.createChunkData(chunk));
}
catch (ClassCastException e)
{
DhLoggerBuilder.getLogger().error("World generator return type incorrect. Error: [" + e.getMessage() + "].", e);
DhLoggerBuilder.getLogger().error("World generator return type incorrect. Error: ["+e.getMessage()+"]. World generator disabled.", e);
Config.Client.WorldGenerator.enableDistantGeneration.set(false);
}
});
@@ -444,31 +447,53 @@ public class WorldGenerationQueue implements Closeable
//==========================//
// executor handler methods //
//==========================//
/**
* Creates a new executor. <br>
* Does nothing if an executor already exists.
*/
public static void setupWorldGenThreadPool()
{
// static setup
if (configListener == null)
{
configListener = new ConfigChangeListener<>(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads, (threadCount) -> { setThreadPoolSize(threadCount); });
}
if (worldGeneratorThreadPool == null || worldGeneratorThreadPool.isTerminated())
{
LOGGER.info("Starting "+ FullDataFileHandler.class.getSimpleName());
setThreadPoolSize(Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get());
}
}
public static void setThreadPoolSize(int threadPoolSize) { worldGeneratorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "DH-Gen-Worker-Thread", Thread.MIN_PRIORITY); }
/**
* Stops any executing tasks and destroys the executor. <br>
* Does nothing if the executor isn't running.
*/
public static void shutdownWorldGenThreadPool()
{
if (worldGeneratorThreadPool != null)
{
LOGGER.info("Stopping "+ FullDataFileHandler.class.getSimpleName());
worldGeneratorThreadPool.shutdownNow();
}
}
//==========//
// shutdown //
//==========//
public CompletableFuture<Void> startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning)
{
queueingThread.shutdownNow();
// remove any incomplete generation tasks
// for (byte detailLevel = QuadTree.TREE_LOWEST_DETAIL_LEVEL; detailLevel < this.waitingTaskQuadTree.treeMaxDetailLevel; detailLevel++)
// {
// MovableGridRingList<WorldGenTask> ringList = this.waitingTaskQuadTree.getRingList(detailLevel);
// ringList.clear((worldGenTask) ->
// {
// if (worldGenTask != null)
// {
// try
// {
// worldGenTask.future.cancel(true);
// }
// catch (CancellationException ignored)
// { /* don't log shutdown exceptions */ }
// }
// });
// }
this.queueingThread.shutdownNow();
// stop and remove any in progress tasks
@@ -519,16 +544,40 @@ public class WorldGenerationQueue implements Closeable
}
LodUtil.assertTrue(this.generatorClosingFuture != null);
LOGGER.info("Awaiting world generator thread pool termination...");
try
{
int waitTimeInSeconds = 3;
if (!worldGeneratorThreadPool.awaitTermination(waitTimeInSeconds, TimeUnit.SECONDS))
{
LOGGER.warn("World generator thread pool shutdown didn't complete after ["+waitTimeInSeconds+"] seconds. Some world generator requests may still be running.");
}
}
catch (InterruptedException e)
{
LOGGER.warn("World generator thread pool shutdown interrupted! Ignoring child threads...", e);
}
this.generator.close();
try
{
this.generatorClosingFuture.cancel(true);
}
catch (Throwable e)
{
LOGGER.error("Failed to close generation queue: ", e);
LOGGER.warn("Failed to close generation queue: ", e);
}
LOGGER.info("Successfully closed "+WorldGenerationQueue.class.getSimpleName());
LOGGER.info("Finished closing "+WorldGenerationQueue.class.getSimpleName());
}
core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvironmentWrapper.java
+5 -4
View File
@@ -24,20 +24,21 @@ import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
public abstract class AbstractBatchGenerationEnvironmentWrapper
{
public AbstractBatchGenerationEnvironmentWrapper(IDhLevel level) { }
public abstract void resizeThreadPool(int newThreadCount);
public abstract void updateAllFutures();
public abstract int getEventCount();
public abstract void stop(boolean blocking);
public abstract void stop();
public abstract CompletableFuture<Void> generateChunks(int minX, int minZ, int genSize, EDhApiWorldGenerationStep targetStep, double runTimeRatio, Consumer<IChunkWrapper> resultConsumer);
public abstract CompletableFuture<Void> generateChunks(
int minX, int minZ, int genSize, EDhApiWorldGenerationStep targetStep,
ExecutorService worldGeneratorThreadPool, Consumer<IChunkWrapper> resultConsumer);
}
core/src/test/java/testItems/worldGeneratorInjection/objects/TestWorldGenerator.java
+2 -1
View File
@@ -7,6 +7,7 @@ import com.seibel.lod.coreapi.DependencyInjection.OverrideInjector;
import com.seibel.lod.core.util.LodUtil;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
/**
@@ -61,7 +62,7 @@ public class TestWorldGenerator implements IDhApiWorldGenerator
public boolean isBusy() { return false; }
@Override
public CompletableFuture<Void> generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode maxGenerationStep, Consumer<Object[]> resultConsumer) { return null; }
public CompletableFuture<Void> generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode maxGenerationStep, ExecutorService executorService, Consumer<Object[]> resultConsumer) { return null; }
@Override
public void preGeneratorTaskStart() { }