From ed935f6a0f90a08968b8c349b348cfd3789af2ad Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 3 Jun 2023 22:29:45 -0500 Subject: [PATCH] Add data transformer and file handler thread configs --- .../com/seibel/lod/core/config/Config.java | 67 ++++++++++++++----- .../transformers/DataRenderTransformer.java | 6 +- .../fullDatafile/FullDataFileHandler.java | 5 +- .../main/resources/assets/lod/lang/en_us.json | 11 ++- 4 files changed, 66 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/config/Config.java b/core/src/main/java/com/seibel/lod/core/config/Config.java index 161386a55..6a6b639c7 100644 --- a/core/src/main/java/com/seibel/lod/core/config/Config.java +++ b/core/src/main/java/com/seibel/lod/core/config/Config.java @@ -696,45 +696,76 @@ public class Config public static class Threading { + public static final String THREAD_NOTE = "" + + " Note: \n" + + " If the total thread count in this config is more threads than your CPU has cores, \n" + + " CPU performance may suffer if Distant Horizons has a lot to load or generate. \n" + + " This can be an issue when first loading into a world, when flying, and/or when generating new terrain."; + + public static final ConfigEntry numberOfWorldGenerationThreads = new ConfigEntry.Builder() .setMinDefaultMax(0.1, - (double) Math.min(Runtime.getRuntime().availableProcessors()/2, 4), + (double) Runtime.getRuntime().availableProcessors()/6, (double) Runtime.getRuntime().availableProcessors()) .comment("" - + " How many threads should be used when generating fake \n" + + " How many threads should be used when generating LOD \n" + " chunks outside the normal render distance? \n" + "\n" + " If it's less than 1, it will be treated as a percentage \n" - + " of time single thread can run before going to idle. \n" + + " of time a single thread can run before going to idle. \n" + "\n" + " If you experience stuttering when generating distant LODs, \n" - + " decrease this number. If you want to increase LOD \n" + + " decrease this number. If you want to increase LOD \n" + " generation speed, increase this number. \n" + "\n" - + " This and the number of buffer builder threads are independent, \n" - + " so if they add up to more threads than your CPU has cores, \n" - + " that shouldn't cause an issue.") + + THREAD_NOTE) .build(); - - + public static ConfigEntry numberOfBufferBuilderThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, - Math.min(Runtime.getRuntime().availableProcessors()/2, 2), + Runtime.getRuntime().availableProcessors()/4, Runtime.getRuntime().availableProcessors()) .comment("" - + "How many threads are used when building vertex buffers? \n" - + " (The things sent to your GPU to draw the fake chunks). \n" + + "How many threads are used when building geometry data? \n" + "\n" - + "If you experience high CPU usage when NOT generating distant \n" + + " If you experience high CPU usage when NOT generating distant \n" + " fake chunks, lower this number. A higher number will make fake\n" + " fake chunks' transition faster when moving around the world. \n" + "\n" - + "This and the number of world generator threads are independent, \n" - + " so if they add up to more threads than your CPU has cores, \n" - + " that shouldn't cause an issue. \n" - + "\n" - + "The maximum value is the number of logical processors on your CPU.") + + THREAD_NOTE) .build(); + + public static final ConfigEntry numberOfFileHandlerThreads = new ConfigEntry.Builder() + .setMinDefaultMax(1, + Runtime.getRuntime().availableProcessors()/8, + Runtime.getRuntime().availableProcessors()) + .comment("" + + " How many threads should be used when reading in LOD data from disk? \n" + + "\n" + + " Increasing this number will cause LODs to load in faster, \n" + + " but may cause stuttering when loading a new world or when \n" + + " quickly flying through existing LODs. \n" + + "\n" + + THREAD_NOTE) + .build(); + + public static final ConfigEntry numberOfDataConverterThreads = new ConfigEntry.Builder() + .setMinDefaultMax(1, + Runtime.getRuntime().availableProcessors()/4, + Runtime.getRuntime().availableProcessors()) + .comment("" + + " How many threads should be used when converting full ID data to render data? \n" + + "\n" + + " These threads run both when terrain is generated and when\n" + + " and when certain graphics settings are changed. \n" + + "\n" + + " Generally this number should be equal to the number of world\n" + + " generator threads, although these threads shouldn't run as\n" + + " often (or as long) as the world generator threads.\n" + + "\n" + + THREAD_NOTE) + .build(); + } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/DataRenderTransformer.java b/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/DataRenderTransformer.java index f71137d8d..1e56c1b28 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/DataRenderTransformer.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/transformers/DataRenderTransformer.java @@ -1,5 +1,6 @@ package com.seibel.lod.core.dataObjects.transformers; +import com.seibel.lod.core.config.Config; import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.lod.core.dataObjects.render.ColumnRenderLoader; import com.seibel.lod.core.dataObjects.render.ColumnRenderSource; @@ -22,6 +23,7 @@ public class DataRenderTransformer private static ExecutorService transformerThreads = null; + //==============// // transformers // //==============// @@ -73,8 +75,8 @@ public class DataRenderTransformer if (transformerThreads == null || transformerThreads.isTerminated()) { LOGGER.info("Starting "+DataRenderTransformer.class.getSimpleName()); - // TODO add config option to set pool size - transformerThreads = ThreadUtil.makeThreadPool(4, "Data/Render Transformer"); + // TODO change when the config is modified + transformerThreads = ThreadUtil.makeThreadPool(Config.Client.Advanced.Threading.numberOfDataConverterThreads.get(), "Data/Render Transformer"); } } diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java index 0a43feb21..84ced0471 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java @@ -1,6 +1,7 @@ package com.seibel.lod.core.file.fullDatafile; import com.google.common.collect.HashMultimap; +import com.seibel.lod.core.config.Config; import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.lod.core.dataObjects.fullData.sources.*; import com.seibel.lod.core.dataObjects.fullData.sources.CompleteFullDataSource; @@ -31,8 +32,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - // TODO add config option to set pool size - protected final ExecutorService fileHandlerThread = ThreadUtil.makeThreadPool(4, FullDataFileHandler.class.getSimpleName()+"Thread"); + // TODO change when config value is changed + protected final ExecutorService fileHandlerThread = ThreadUtil.makeThreadPool(Config.Client.Advanced.Threading.numberOfFileHandlerThreads.get(), FullDataFileHandler.class.getSimpleName()+"Thread"); protected final ConcurrentHashMap fileBySectionPos = new ConcurrentHashMap<>(); protected final IDhLevel level; protected final File saveDir; diff --git a/core/src/main/resources/assets/lod/lang/en_us.json b/core/src/main/resources/assets/lod/lang/en_us.json index d8a61d9bb..1b9abb7b0 100644 --- a/core/src/main/resources/assets/lod/lang/en_us.json +++ b/core/src/main/resources/assets/lod/lang/en_us.json @@ -257,6 +257,7 @@ "When matching worlds of the same dimension type the\ntested chunks must be at least this percent the same\nin order to be considered the same world.\n\nNote: If you use portals to enter a dimension at two\ndifferent locations this system may think it is two different worlds.\n\n§61.0:§r the chunks must be identical.\n§60.5:§r the chunks must be half the same.\n§60.0:§r disables multi-dimension support\n only one world will be used per dimension.", "lod.config.client.advanced": "Advanced options", + "lod.config.client.advanced.threading": "Threading", "lod.config.client.advanced.threading.numberOfWorldGenerationThreads": @@ -266,7 +267,15 @@ "lod.config.client.advanced.threading.numberOfBufferBuilderThreads": "NO. of buffer builder threads", "lod.config.client.advanced.threading.numberOfBufferBuilderThreads.@tooltip": - "The number of threads used when building vertex buffers\n(The things sent to your GPU to draw the fake chunks).\nCan only be between 1 and your CPU's processor count.", + "The number of threads used when building geometry data.\nCan only be between 1 and your CPU's processor count.", + "lod.config.client.advanced.threading.numberOfFileHandlerThreads": + "NO. of file handler threads", + "lod.config.client.advanced.threading.numberOfFileHandlerThreads.@tooltip": + "The number of threads used when building vertex buffers\n(The things sent to your GPU to draw the fake chunks).\nCan only be between 1 and your CPU's processor count.", + "lod.config.client.advanced.threading.numberOfDataConverterThreads": + "NO. of data converter threads", + "lod.config.client.advanced.threading.numberOfDataConverterThreads.@tooltip": + "The number of threads used when converting ID data to render-able data.\n(This generally happens when generating new terrain or changing graphics settings).\nCan only be between 1 and your CPU's processor count.", "lod.config.client.advanced.debugging": "Debug", "lod.config.client.advanced.debugging.rendererMode":