From d502fd4daa8e32e737849159cdd003ed2b2544af Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 14 Jun 2023 19:42:34 -0500 Subject: [PATCH] Add a thread config for ChunkToLodBuilder --- .../com/seibel/lod/core/config/Config.java | 13 + .../ThreadPresetConfigEventHandler.java | 10 + .../transformers/ChunkToLodBuilder.java | 265 +++++++++++------- .../lod/core/level/AbstractDhClientLevel.java | 2 +- .../main/resources/assets/lod/lang/en_us.json | 14 +- 5 files changed, 204 insertions(+), 100 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/config/Config.java b/core/src/main/java/com/seibel/lod/core/config/Config.java index 88981d2ed..581b5761c 100644 --- a/core/src/main/java/com/seibel/lod/core/config/Config.java +++ b/core/src/main/java/com/seibel/lod/core/config/Config.java @@ -805,6 +805,19 @@ public class Config + THREAD_NOTE) .build(); + public static final ConfigEntry numberOfChunkLodConverterThreads = new ConfigEntry.Builder() + .setMinDefaultMax(1, + Runtime.getRuntime().availableProcessors()/16, + Runtime.getRuntime().availableProcessors()) + .comment("" + + "How many threads should be used to convert Minecraft chunks into LOD data? \n" + + "\n" + + "These threads run both when terrain is generated and when\n" + + "chunks are loaded, unloaded, and modified. \n" + + "\n" + + THREAD_NOTE) + .build(); + } public static class GpuBuffers diff --git a/core/src/main/java/com/seibel/lod/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/lod/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index 91816b57e..7946f9309 100644 --- a/core/src/main/java/com/seibel/lod/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/lod/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -55,6 +55,15 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + private final ConfigEntryWithPresetOptions chunkLodConverters = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 1); + this.put(EThreadPreset.LOW_IMPACT, getThreadCountByPercent(0.1)); + this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.2)); + this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); + }}); @@ -70,6 +79,7 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.configList.add(this.bufferBuilders); this.configList.add(this.fileHandlers); this.configList.add(this.dataConverters); + this.configList.add(this.chunkLodConverters); for (ConfigEntryWithPresetOptions config : this.configList) diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/ChunkToLodBuilder.java index 833171c60..b9680e169 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -1,8 +1,11 @@ package com.seibel.lod.core.dataObjects.transformers; +import java.io.Closeable; +import java.io.IOException; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import com.seibel.lod.core.config.listeners.ConfigChangeListener; import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.lod.core.config.Config; import com.seibel.lod.core.logging.ConfigBasedLogger; @@ -13,66 +16,65 @@ import com.seibel.lod.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.lod.core.dependencyInjection.SingletonInjector; import org.apache.logging.log4j.LogManager; -//FIXME: To-Be-Used class -public class ChunkToLodBuilder +public class ChunkToLodBuilder implements AutoCloseable { - public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(), () -> Config.Client.Advanced.Logging.logLodBuilderEvent.get()); + public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(), () -> Config.Client.Advanced.Logging.logLodBuilderEvent.get()); private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); - public static final long MAX_TICK_TIME_NS = 1000000000L / 20L; - public static final int THREAD_COUNT = 1; + public static final long MAX_TICK_TIME_NS = 1000000000L / 20L; - private static class Task + + private final ConcurrentHashMap latestChunkToBuild = new ConcurrentHashMap<>(); + private final ConcurrentLinkedDeque taskToBuild = new ConcurrentLinkedDeque<>(); + private final AtomicInteger runningCount = new AtomicInteger(0); + + private int threadCount = -1; + private ExecutorService executorThreadPool = null; + private ConfigChangeListener configListener; + + + + //==============// + // constructors // + //==============// + + public ChunkToLodBuilder() { this.setupExecutorService(); } + + + + //=================// + // data generation // + //=================// + + public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) { - final DhChunkPos chunkPos; - final CompletableFuture future; - - Task(DhChunkPos chunkPos, CompletableFuture future) - { - this.chunkPos = chunkPos; - this.future = future; - } - } - private final ConcurrentHashMap latestChunkToBuild = new ConcurrentHashMap<>(); - private final ConcurrentLinkedDeque taskToBuild = new ConcurrentLinkedDeque<>(); - private final ExecutorService executor = ThreadUtil.makeThreadPool(THREAD_COUNT, ChunkToLodBuilder.class); - private final AtomicInteger runningCount = new AtomicInteger(0); - - - - public ChunkToLodBuilder() { } - - - - public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) - { - if (chunkWrapper == null) + if (chunkWrapper == null) { throw new NullPointerException("ChunkWrapper cannot be null!"); } - IChunkWrapper oldChunk = this.latestChunkToBuild.put(chunkWrapper.getChunkPos(), chunkWrapper); // an Exchange operation - // If there's old chunk, that means we just replaced an unprocessed old request on generating data on this pos. - // if so, we can just return null to signal this, as the old request's future will instead be the proper one - // that will return the latest generated data. - if (oldChunk != null) + IChunkWrapper oldChunk = this.latestChunkToBuild.put(chunkWrapper.getChunkPos(), chunkWrapper); // an Exchange operation + // If there's old chunk, that means we just replaced an unprocessed old request on generating data on this pos. + // if so, we can just return null to signal this, as the old request's future will instead be the proper one + // that will return the latest generated data. + if (oldChunk != null) { return null; } - // Otherwise, it means we're the first to do so. Let's submit our task to this entry. - CompletableFuture future = new CompletableFuture<>(); + // Otherwise, it means we're the first to do so. Let's submit our task to this entry. + CompletableFuture future = new CompletableFuture<>(); this.taskToBuild.addLast(new Task(chunkWrapper.getChunkPos(), future)); - return future; - } + return future; + } - public void tick() + public void tick() { - if (this.runningCount.get() >= THREAD_COUNT) + if (this.runningCount.get() >= this.threadCount) { return; } - else if (this.taskToBuild.isEmpty()) + else if (this.taskToBuild.isEmpty()) { return; } @@ -86,95 +88,94 @@ public class ChunkToLodBuilder } - for (int i = 0; i + CompletableFuture.runAsync(() -> { - try + try { - _tick(); - } + this._tick(); + } finally { this.runningCount.decrementAndGet(); - } - }, this.executor); - } - } - private void _tick() + } + }, this.executorThreadPool); + } + } + private void _tick() { - long time = System.nanoTime(); - int count = 0; - boolean allDone = false; - while (true) + long time = System.nanoTime(); + int count = 0; + boolean allDone = false; + while (true) { // run until we either run out of time, or all tasks are complete - if (System.nanoTime() - time > MAX_TICK_TIME_NS && !this.taskToBuild.isEmpty()) + if (System.nanoTime() - time > MAX_TICK_TIME_NS && !this.taskToBuild.isEmpty()) { break; } - Task task = this.taskToBuild.pollFirst(); - if (task == null) + Task task = this.taskToBuild.pollFirst(); + if (task == null) { - allDone = true; - break; - } + allDone = true; + break; + } - count++; - IChunkWrapper latestChunk = this.latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation - if (latestChunk == null) + count++; + IChunkWrapper latestChunk = this.latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation + if (latestChunk == null) { - LOGGER.error("Somehow Task at "+task.chunkPos+" has latestChunk as null. Skipping task."); - task.future.complete(null); - continue; - } + LOGGER.error("Somehow Task at "+task.chunkPos+" has latestChunk as null. Skipping task."); + task.future.complete(null); + continue; + } - try + try { - if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) + if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { - ChunkSizedFullDataAccessor data = LodDataBuilder.createChunkData(latestChunk); - if (data != null) + ChunkSizedFullDataAccessor data = LodDataBuilder.createChunkData(latestChunk); + if (data != null) { - task.future.complete(data); - continue; - } - } - } + task.future.complete(data); + continue; + } + } + } catch (Exception ex) { - LOGGER.error("Error while processing Task at "+task.chunkPos, ex); - } + LOGGER.error("Error while processing Task at "+task.chunkPos, ex); + } - // Failed to build due to chunk not meeting requirement. - IChunkWrapper casChunk = this.latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null - if (casChunk == null || latestChunk.isStillValid()) // That means CAS have been successful + // Failed to build due to chunk not meeting requirement. + IChunkWrapper casChunk = this.latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null + if (casChunk == null || latestChunk.isStillValid()) // That means CAS have been successful { this.taskToBuild.addLast(task); // Then add back the same old task. } - else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. + else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. { task.future.complete(null); } - count--; - } + count--; + } - long time2 = System.nanoTime(); - if (!allDone) + long time2 = System.nanoTime(); + if (!allDone) { - //LOGGER.info("Completed {} tasks in {} in this tick", count, Duration.ofNanos(time2 - time)); - } + //LOGGER.info("Completed {} tasks in {} in this tick", count, Duration.ofNanos(time2 - time)); + } else if (count > 0) { - //LOGGER.info("Completed all {} tasks in {}", count, Duration.ofNanos(time2 - time)); - } - } + //LOGGER.info("Completed all {} tasks in {}", count, Duration.ofNanos(time2 - time)); + } + } - - /** + /** * should be called whenever changing levels/worlds * to prevent trying to generate LODs for chunk(s) that are no longer loaded * (which can cause exceptions) @@ -185,4 +186,80 @@ public class ChunkToLodBuilder this.latestChunkToBuild.clear(); } + + + //==========================// + // executor handler methods // + //==========================// + + /** + * Creates a new executor.
+ * Does nothing if an executor already exists. + */ + public void setupExecutorService() + { + // static setup + if (this.configListener == null) + { + this.configListener = new ConfigChangeListener<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads, (threadCount) -> { this.setThreadPoolSize(threadCount); }); + } + + + if (this.executorThreadPool == null || this.executorThreadPool.isTerminated()) + { + LOGGER.info("Starting "+ChunkToLodBuilder.class.getSimpleName()); + this.setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads.get()); + } + } + public void setThreadPoolSize(int threadPoolSize) + { + this.threadCount = threadPoolSize; + this.executorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, ChunkToLodBuilder.class); + } + + /** + * Stops any executing tasks and destroys the executor.
+ * Does nothing if the executor isn't running. + */ + public void shutdownExecutorService() + { + if (this.executorThreadPool != null) + { + LOGGER.info("Stopping "+ChunkToLodBuilder.class.getSimpleName()); + this.executorThreadPool.shutdownNow(); + } + } + + + + //==============// + // base methods // + //==============// + + @Override + public void close() + { + this.shutdownExecutorService(); + this.clearCurrentTasks(); + } + + + + //================// + // helper classes // + //================// + + private static class Task + { + final DhChunkPos chunkPos; + final CompletableFuture future; + + Task(DhChunkPos chunkPos, CompletableFuture future) + { + this.chunkPos = chunkPos; + this.future = future; + } + } + } + diff --git a/core/src/main/java/com/seibel/lod/core/level/AbstractDhClientLevel.java b/core/src/main/java/com/seibel/lod/core/level/AbstractDhClientLevel.java index fb344f714..4ca81efc5 100644 --- a/core/src/main/java/com/seibel/lod/core/level/AbstractDhClientLevel.java +++ b/core/src/main/java/com/seibel/lod/core/level/AbstractDhClientLevel.java @@ -218,7 +218,7 @@ public abstract class AbstractDhClientLevel implements IDhClientLevel this.fullDataFileHandler.close(); // clear the chunk builder to prevent generating LODs for chunks that are unloaded - this.chunkToLodBuilder.clearCurrentTasks(); + this.chunkToLodBuilder.close(); // shutdown the renderer diff --git a/core/src/main/resources/assets/lod/lang/en_us.json b/core/src/main/resources/assets/lod/lang/en_us.json index 8259312af..fd591999b 100644 --- a/core/src/main/resources/assets/lod/lang/en_us.json +++ b/core/src/main/resources/assets/lod/lang/en_us.json @@ -308,15 +308,19 @@ "lod.config.client.advanced.multiThreading.numberOfBufferBuilderThreads": "NO. of buffer builder threads", "lod.config.client.advanced.multiThreading.numberOfBufferBuilderThreads.@tooltip": - "The number of threads used when building geometry data.\nCan only be between 1 and your CPU's processor count.", + "The number of threads used when building geometry data. \nCan only be between 1 and your CPU's processor count.", "lod.config.client.advanced.multiThreading.numberOfFileHandlerThreads": - "NO. of file handler threads", + "NO. of file handler threads", "lod.config.client.advanced.multiThreading.numberOfFileHandlerThreads.@tooltip": - "The number of threads used when building vertex buffers\n(The things sent to your GPU to draw the LODs).\nCan only be between 1 and your CPU's processor count.", + "The number of threads used when building vertex buffers \n(The things sent to your GPU to draw the LODs). \nCan only be between 1 and your CPU's processor count.", "lod.config.client.advanced.multiThreading.numberOfDataConverterThreads": - "NO. of data converter threads", + "NO. of data converter threads", "lod.config.client.advanced.multiThreading.numberOfDataConverterThreads.@tooltip": - "The number of threads used when converting ID data to render-able data.\n(This generally happens when generating new terrain or changing graphics settings).\nCan only be between 1 and your CPU's processor count.", + "The number of threads used when converting ID data to render-able data. \n(This generally happens when generating new terrain or changing graphics settings). \nCan only be between 1 and your CPU's processor count.", + "lod.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads": + "NO. of chunk LOD converter threads", + "lod.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads.@tooltip": + "How many threads should be used to convert Minecraft chunks into LOD data? \nThese threads run both when terrain is generated and when \nchunks are loaded, unloaded, and modified.",