From 24b32ad6c980b3349bc1ed0eacbcf5b65831483d Mon Sep 17 00:00:00 2001 From: coolGi Date: Mon, 17 Jul 2023 01:32:38 +0930 Subject: [PATCH 01/10] Slight changes to the api fabric.mod.json to make it look like a library in modmenu --- api/src/main/resources/fabric.mod.json | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/src/main/resources/fabric.mod.json b/api/src/main/resources/fabric.mod.json index 263109227..01c9a7321 100644 --- a/api/src/main/resources/fabric.mod.json +++ b/api/src/main/resources/fabric.mod.json @@ -4,7 +4,15 @@ "version": "1.0", "name": "DhApi", - "description": "This is just a placeholder to make uploading to Modrinth happy. Don't try running this jar in Minecraft. \nNote: while the API mod appears in mod menu lists when running DH via gradle, it won't appear when DH has been built and run in retail MC." - + "description": "This is just a placeholder to make uploading to Modrinth happy. Don't try running this jar in Minecraft. \nNote: while the API mod appears in mod menu lists when running DH via gradle, it won't appear when DH has been built and run in retail MC.", + + "custom": { + "modmenu": { + "badges": ["library"], + "parent": { + "id": "distanthorizons" + } + } + } } From 3190d607294b0136f2089c7c6410e0738844a813 Mon Sep 17 00:00:00 2001 From: coolGi Date: Mon, 17 Jul 2023 02:08:34 +0930 Subject: [PATCH 02/10] Slight fix to the reliability of the auto updater --- .../distanthorizons/core/config/Config.java | 2 +- .../core/jar/updater/SelfUpdater.java | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 05f70c537..2dbfebf26 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -878,7 +878,7 @@ public class Config .build(); public static ConfigEntry enableSilentUpdates = new ConfigEntry.Builder() - .set(true) + .set(false) .comment("" + "Should Distant Horizons silently, automatically download and install new versions? " + "") diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java index 54fa9a906..ec5b91592 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java @@ -10,7 +10,10 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.wrapperInterfaces.IVersionConstants; import org.apache.logging.log4j.Logger; +import java.io.File; +import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; import java.security.MessageDigest; /** @@ -23,6 +26,7 @@ public class SelfUpdater { /** As we cannot delete(or replace) the jar while the mod is running, we just have this to delete it once the game closes */ public static boolean deleteOldOnClose = false; + public static File newFileLocation; /** * Should be called on the game starting. @@ -49,8 +53,9 @@ public class SelfUpdater { LOGGER.info("New version ("+ModrinthGetter.getLatestNameForVersion(mcVersion)+") of "+ ModInfo.READABLE_NAME+" is available"); if (Config.Client.Advanced.AutoUpdater.enableSilentUpdates.get()) { + newFileLocation = JarUtils.jarFile.getParentFile().toPath().resolve("update").resolve(ModInfo.NAME + "-" + ModrinthGetter.getLatestNameForVersion(mcVersion) + ".jar").toFile(); // Auto-update mod - updateMod(mcVersion); + updateMod(mcVersion, newFileLocation); return false; } // else return true; @@ -62,6 +67,13 @@ public class SelfUpdater { */ public static void onClose() { if (deleteOldOnClose) { + try { + Files.move(newFileLocation.toPath(), JarUtils.jarFile.getParentFile().toPath().resolve(newFileLocation.getName())); + Files.delete(newFileLocation.getParentFile().toPath()); + } catch (Exception e) { + LOGGER.warn("Failed to move updated fire from ["+ newFileLocation.getAbsolutePath() +"] to ["+ JarUtils.jarFile.getParentFile().getAbsolutePath() +"], please move it manually"); + e.printStackTrace(); + } try { Files.delete(JarUtils.jarFile.toPath()); } catch (Exception e) { @@ -72,14 +84,18 @@ public class SelfUpdater { } public static boolean updateMod() { + String mcVer = SingletonInjector.INSTANCE.get(IVersionConstants.class).getMinecraftVersion(); + newFileLocation = JarUtils.jarFile.getParentFile().toPath().resolve("update").resolve(ModInfo.NAME + "-" + ModrinthGetter.getLatestNameForVersion(mcVer) + ".jar").toFile(); return updateMod( - SingletonInjector.INSTANCE.get(IVersionConstants.class).getMinecraftVersion() + mcVer, + newFileLocation ); } - public static boolean updateMod(String minecraftVersion) { + public static boolean updateMod(String minecraftVersion, File file) { try { LOGGER.info("Attempting to auto update " + ModInfo.READABLE_NAME); - WebDownloader.downloadAsFile(ModrinthGetter.getLatestDownloadForVersion(minecraftVersion), JarUtils.jarFile.getParentFile().toPath().resolve(ModInfo.NAME + "-" + ModrinthGetter.getLatestNameForVersion(minecraftVersion) + ".jar").toFile()); + Files.createDirectories(file.getParentFile().toPath()); + WebDownloader.downloadAsFile(ModrinthGetter.getLatestDownloadForVersion(minecraftVersion), file); deleteOldOnClose = true; LOGGER.info(ModInfo.READABLE_NAME + " successfully updated. It will apply on game's relaunch"); return true; From f6e68082b71d91c8b7ed04c27e3215ec09cb5ceb Mon Sep 17 00:00:00 2001 From: coolGi Date: Mon, 17 Jul 2023 02:14:17 +0930 Subject: [PATCH 03/10] Added checking for the checksum of the update --- .../distanthorizons/core/jar/updater/SelfUpdater.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java index ec5b91592..d08f3efa3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java @@ -94,13 +94,22 @@ public class SelfUpdater { public static boolean updateMod(String minecraftVersion, File file) { try { LOGGER.info("Attempting to auto update " + ModInfo.READABLE_NAME); + Files.createDirectories(file.getParentFile().toPath()); WebDownloader.downloadAsFile(ModrinthGetter.getLatestDownloadForVersion(minecraftVersion), file); + + // Check if the checksum of the downloaded jar is correct (not required, but good to have to prevent corruption or interception) + if (!JarUtils.getFileChecksum(MessageDigest.getInstance("SHA"), file).equals(ModrinthGetter.getLatestShaForVersion(minecraftVersion))) { + LOGGER.warn("DH update checksum failed, aborting install"); + throw new Exception("Checksum failed"); + } + deleteOldOnClose = true; + LOGGER.info(ModInfo.READABLE_NAME + " successfully updated. It will apply on game's relaunch"); return true; } catch (Exception e) { - LOGGER.info("Failed to update "+ModInfo.READABLE_NAME+" to version "+ModrinthGetter.getLatestNameForVersion(minecraftVersion)); + LOGGER.warn("Failed to update "+ModInfo.READABLE_NAME+" to version "+ModrinthGetter.getLatestNameForVersion(minecraftVersion)); e.printStackTrace(); return false; } From fe16c5a9cacf8200640b642f5830359b0b38bf57 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 13:38:00 -0500 Subject: [PATCH 04/10] Fix old thread pools not being shutdown when resized --- .../bufferBuilding/ColumnRenderBufferBuilder.java | 6 ++++++ .../transformers/DataRenderTransformer.java | 11 ++++++++++- .../core/file/fullDatafile/FullDataFileHandler.java | 11 ++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 9b788b029..27b636a19 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -387,6 +387,12 @@ public class ColumnRenderBufferBuilder } public static void setThreadPoolSize(int threadPoolSize) { + if (bufferBuilderThreadPool != null) + { + // close the previous thread pool if one exists + bufferBuilderThreadPool.shutdown(); + } + bufferBuilderThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Buffer Builder"); maxNumberOfConcurrentCalls = threadPoolSize * MAX_NUMBER_OF_CONCURRENT_CALLS_PER_THREAD; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java index 5b339d2a3..a51f7c9fc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java @@ -87,7 +87,16 @@ public class DataRenderTransformer setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads.get()); } } - public static void setThreadPoolSize(int threadPoolSize) { transformerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Full/Render Data Transformer"); } + public static void setThreadPoolSize(int threadPoolSize) + { + if (transformerThreadPool != null) + { + // close the previous thread pool if one exists + transformerThreadPool.shutdown(); + } + + transformerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Full/Render Data Transformer"); + } /** * Stops any executing tasks and destroys the executor.
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 66d5fcd4c..f81efb3ee 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -578,7 +578,16 @@ public class FullDataFileHandler implements IFullDataSourceProvider setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get()); } } - public static void setThreadPoolSize(int threadPoolSize) { fileHandlerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread"); } + public static void setThreadPoolSize(int threadPoolSize) + { + if (fileHandlerThreadPool != null) + { + // close the previous thread pool if one exists + fileHandlerThreadPool.shutdown(); + } + + fileHandlerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread"); + } /** * Stops any executing tasks and destroys the executor.
From c3ed5f98a5c13f0286fecdacc663379540d3141b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 17:05:45 -0500 Subject: [PATCH 05/10] Add and use Rate Limited thread pools --- .../distanthorizons/core/config/Config.java | 25 ++++++++ .../ColumnRenderBufferBuilder.java | 2 +- .../transformers/ChunkToLodBuilder.java | 2 +- .../transformers/DataRenderTransformer.java | 2 +- .../fullDatafile/FullDataFileHandler.java | 2 +- .../core/generation/WorldGenerationQueue.java | 11 +++- .../distanthorizons/core/util/ThreadUtil.java | 62 +++++++++--------- .../RateLimitedThreadPoolExecutor.java | 63 +++++++++++++++++++ .../assets/distanthorizons/lang/en_us.json | 14 +++++ 9 files changed, 150 insertions(+), 33 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 2dbfebf26..942063ace 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -737,6 +737,11 @@ public class Config + "CPU performance may suffer if Distant Horizons has a lot to load or generate. \n" + "This can be an issue when first loading into a world, when flying, and/or when generating new terrain."; + public static final String THREAD_RUN_TIME_RATIO_NOTE = "" + + "If this value is less than 1.0, it will be treated as a percentage \n" + + "of time a each thread can run before going idle. \n" + + ""; + public static final ConfigEntry numberOfWorldGenerationThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -753,6 +758,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForWorldGenerationThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static ConfigEntry numberOfBufferBuilderThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -767,6 +776,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForBufferBuilderThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static final ConfigEntry numberOfFileHandlerThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -781,6 +794,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForFileHandlerThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static final ConfigEntry numberOfDataConverterThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -798,6 +815,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForDataConverterThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static final ConfigEntry numberOfChunkLodConverterThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -811,6 +832,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForChunkLodConverterThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 27b636a19..d90cbec5e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -393,7 +393,7 @@ public class ColumnRenderBufferBuilder bufferBuilderThreadPool.shutdown(); } - bufferBuilderThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Buffer Builder"); + bufferBuilderThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "Buffer Builder", Config.Client.Advanced.MultiThreading.runTimeRatioForBufferBuilderThreads); maxNumberOfConcurrentCalls = threadPoolSize * MAX_NUMBER_OF_CONCURRENT_CALLS_PER_THREAD; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java index 5931b2ab1..5fde3959f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -230,7 +230,7 @@ public class ChunkToLodBuilder implements AutoCloseable } threadCount = threadPoolSize; - executorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, ChunkToLodBuilder.class); + executorThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, ChunkToLodBuilder.class.getSimpleName(), Config.Client.Advanced.MultiThreading.runTimeRatioForChunkLodConverterThreads); } /** diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java index a51f7c9fc..6f3802754 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java @@ -95,7 +95,7 @@ public class DataRenderTransformer transformerThreadPool.shutdown(); } - transformerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Full/Render Data Transformer"); + transformerThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "Full/Render Data Transformer", Config.Client.Advanced.MultiThreading.runTimeRatioForDataConverterThreads); } /** diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index f81efb3ee..668a03cbf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -586,7 +586,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider fileHandlerThreadPool.shutdown(); } - fileHandlerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread"); + fileHandlerThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread", Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads); } /** diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 2e1a1ad1d..228990d6c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -510,7 +510,16 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads.get()); } } - public static void setThreadPoolSize(int threadPoolSize) { worldGeneratorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "DH-Gen-Worker-Thread", Thread.MIN_PRIORITY); } + public static void setThreadPoolSize(int threadPoolSize) + { + if (worldGeneratorThreadPool != null) + { + // close the previous thread pool if one exists + worldGeneratorThreadPool.shutdown(); + } + + worldGeneratorThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "DH-Gen-Worker-Thread", Thread.MIN_PRIORITY, Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads); + } /** * Stops any executing tasks and destroys the executor.
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java index 9e17a82d0..19fb369d2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java @@ -1,6 +1,9 @@ package com.seibel.distanthorizons.core.util; +import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; +import com.seibel.distanthorizons.core.config.types.ConfigEntry; import com.seibel.distanthorizons.core.util.objects.DhThreadFactory; +import com.seibel.distanthorizons.core.util.objects.RateLimitedThreadPoolExecutor; import java.util.concurrent.*; @@ -9,9 +12,33 @@ public class ThreadUtil public static int MINIMUM_RELATIVE_PRIORITY = -5; public static int DEFAULT_RELATIVE_PRIORITY = 0; + // TODO currently only used for RateLimitedThreadPools could this be used for all thread pools? + /** used to track and remove old listeners for certain pools if the thread pool is recreated. */ + private static final ConcurrentHashMap> THREAD_CHANGE_LISTENERS_BY_THREAD_NAME = new ConcurrentHashMap<>(); + // create rate limited thread pool // + + public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, ConfigEntry runTimeRatioConfigEntry) { return makeRateLimitedThreadPool(poolSize, name, 0, runTimeRatioConfigEntry); } + public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, int relativePriority, ConfigEntry runTimeRatioConfigEntry) + { + // remove the old listener if one exists + if (THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.containsKey(name)) + { + // note: this assumes only one thread pool exists with a given name + THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.get(name).close(); + THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.remove(name); + } + + RateLimitedThreadPoolExecutor executor = new RateLimitedThreadPoolExecutor(poolSize, runTimeRatioConfigEntry.get(), new DhThreadFactory("DH-" + name, Thread.NORM_PRIORITY+relativePriority)); + + ConfigChangeListener changeListener = new ConfigChangeListener<>(runTimeRatioConfigEntry, (newRunTimeRatio) -> { executor.runTimeRatio = newRunTimeRatio; }); + THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.put(name, changeListener); + + return executor; + } + // create thread pool // @@ -26,38 +53,17 @@ public class ThreadUtil new DhThreadFactory("DH-" + name, Thread.NORM_PRIORITY+relativePriority)); } - public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz, int relativePriority) - { - return makeThreadPool(poolSize, clazz.getSimpleName(), relativePriority); - } - public static ThreadPoolExecutor makeThreadPool(int poolSize, String name) - { - return makeThreadPool(poolSize, name, 0); - } - public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz) - { - return makeThreadPool(poolSize, clazz.getSimpleName(), 0); - } + public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz, int relativePriority) { return makeThreadPool(poolSize, clazz.getSimpleName(), relativePriority); } + public static ThreadPoolExecutor makeThreadPool(int poolSize, String name) { return makeThreadPool(poolSize, name, 0); } + public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz) { return makeThreadPool(poolSize, clazz.getSimpleName(), 0); } // create single thread pool // - public static ThreadPoolExecutor makeSingleThreadPool(String name, int relativePriority) - { - return makeThreadPool(1, name, relativePriority); - } - public static ThreadPoolExecutor makeSingleThreadPool(Class clazz, int relativePriority) - { - return makeThreadPool(1, clazz.getSimpleName(), relativePriority); - } - public static ThreadPoolExecutor makeSingleThreadPool(String name) - { - return makeThreadPool(1, name, 0); - } - public static ThreadPoolExecutor makeSingleThreadPool(Class clazz) - { - return makeThreadPool(1, clazz.getSimpleName(), 0); - } + public static ThreadPoolExecutor makeSingleThreadPool(String name, int relativePriority) { return makeThreadPool(1, name, relativePriority); } + public static ThreadPoolExecutor makeSingleThreadPool(Class clazz, int relativePriority) { return makeThreadPool(1, clazz.getSimpleName(), relativePriority); } + public static ThreadPoolExecutor makeSingleThreadPool(String name) { return makeThreadPool(1, name, 0); } + public static ThreadPoolExecutor makeSingleThreadPool(Class clazz) { return makeThreadPool(1, clazz.getSimpleName(), 0); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java new file mode 100644 index 000000000..7b23a6dc4 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java @@ -0,0 +1,63 @@ +package com.seibel.distanthorizons.core.util.objects; + +import java.util.concurrent.*; + +/** + * Can be used to more finely control CPU usage and + * reduce CPU usage if only 1 thread is already assigned. + */ +public class RateLimitedThreadPoolExecutor extends ThreadPoolExecutor +{ + public volatile double runTimeRatio; + + /** When this thread started running its last task */ + private final ThreadLocal runStartNanoTimeRef = ThreadLocal.withInitial(() -> -1L); + /** How long it took this thread to run its last task */ + private final ThreadLocal lastRunDurationNanoTimeRef = ThreadLocal.withInitial(() -> -1L); + + + + //==============// + // constructors // + //==============// + + public RateLimitedThreadPoolExecutor(int corePoolSize, double runTimeRatio, ThreadFactory threadFactory) + { + super(corePoolSize, corePoolSize, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + threadFactory); + + this.runTimeRatio = runTimeRatio; + } + + + + //===========// + // overrides // + //===========// + + protected void beforeExecute(Thread thread, Runnable runnable) + { + super.beforeExecute(thread, runnable); + + if (this.runTimeRatio < 1.0 && this.lastRunDurationNanoTimeRef.get() != -1) + { + try + { + long deltaMs = TimeUnit.NANOSECONDS.toMillis(this.lastRunDurationNanoTimeRef.get()); + Thread.sleep((long) (deltaMs/this.runTimeRatio - deltaMs)); + } catch (InterruptedException ignored) { } + } + + this.runStartNanoTimeRef.set(System.nanoTime()); + } + + @Override + protected void afterExecute(Runnable runnable, Throwable throwable) + { + super.afterExecute(runnable, throwable); + this.lastRunDurationNanoTimeRef.set(System.nanoTime() - this.runStartNanoTimeRef.get()); + } + +} \ No newline at end of file diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 94d2bbc46..3ee414af4 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -306,22 +306,36 @@ "NO. of world generation threads", "distanthorizons.config.client.advanced.multiThreading.numberOfWorldGenerationThreads.@tooltip": "How many threads should be used when generating LODs \n outside the normal render distance? \n\nIf it this is less than 1, it will be treated as a percentage \nof time a single thread can run before going idle. \n\nIf you experience stuttering when generating distant LODs, \ndecrease this number. If you want to increase LOD \ngeneration speed, increase this number. \n\nThis and the number of buffer builder threads are independent, \nif they add up to more threads than your CPU has cores \nthat is ok.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForWorldGenerationThreads": + "Runtime % for world generation threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfBufferBuilderThreads": "NO. of buffer builder threads", "distanthorizons.config.client.advanced.multiThreading.numberOfBufferBuilderThreads.@tooltip": "The number of threads used when building geometry data. \nCan only be between 1 and your CPU's processor count.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForBufferBuilderThreads": + "Runtime % for buffer builder threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfFileHandlerThreads": "NO. of file handler threads", "distanthorizons.config.client.advanced.multiThreading.numberOfFileHandlerThreads.@tooltip": "The number of threads used when building vertex buffers \n(The things sent to your GPU to draw the LODs). \nCan only be between 1 and your CPU's processor count.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForFileHandlerThreads": + "Runtime % for file handler threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfDataConverterThreads": "NO. of data converter threads", "distanthorizons.config.client.advanced.multiThreading.numberOfDataConverterThreads.@tooltip": "The number of threads used when converting ID data to render-able data. \n(This generally happens when generating new terrain or changing graphics settings). \nCan only be between 1 and your CPU's processor count.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForDataConverterThreads": + "Runtime % for data converter threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads": "NO. of chunk LOD converter threads", "distanthorizons.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads.@tooltip": "How many threads should be used to convert Minecraft chunks into LOD data? \nThese threads run both when terrain is generated and when \nchunks are loaded, unloaded, and modified.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForChunkLodConverterThreads": + "Runtime % for chunk LOD converter threads", From a7a54598b29dcc035f6dd214c62df43370d967b9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 17:41:32 -0500 Subject: [PATCH 06/10] Add Thread runTime to the thread presets --- .../distanthorizons/core/config/Config.java | 19 ++-- .../ThreadPresetConfigEventHandler.java | 90 ++++++++++++++++--- 2 files changed, 88 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 942063ace..c9c98cb98 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -739,8 +739,11 @@ public class Config public static final String THREAD_RUN_TIME_RATIO_NOTE = "" + "If this value is less than 1.0, it will be treated as a percentage \n" - + "of time a each thread can run before going idle. \n" - + ""; + + "of time each thread can run before going idle. \n" + + "\n" + + "This can be used to reduce CPU usage if the thread count \n" + + "is already set to 1 for the given option, or more finely \n" + + "tune CPU performance."; public static final ConfigEntry numberOfWorldGenerationThreads = new ConfigEntry.Builder() @@ -759,7 +762,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForWorldGenerationThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getWorldGenDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); @@ -777,7 +780,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForBufferBuilderThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getBufferBuilderDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); @@ -795,7 +798,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForFileHandlerThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getFileHandlerDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); @@ -816,13 +819,13 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForDataConverterThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getDataConverterDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); public static final ConfigEntry numberOfChunkLodConverterThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, - ThreadPresetConfigEventHandler.getChunkLodConvertersDefaultThreadCount(), + ThreadPresetConfigEventHandler.getChunkLodConverterDefaultThreadCount(), Runtime.getRuntime().availableProcessors()) .comment("" + "How many threads should be used to convert Minecraft chunks into LOD data? \n" @@ -833,7 +836,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForChunkLodConverterThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getChunkLodConverterDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index 2c60d3ea9..5bdbc3722 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -20,8 +20,9 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan private static final Logger LOGGER = LogManager.getLogger(); + public static int getWorldGenDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions worldGen = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, + private final ConfigEntryWithPresetOptions worldGenThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -30,9 +31,20 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getWorldGenDefaultRunTimeRatio() { return 0.25; } + private final ConfigEntryWithPresetOptions worldGenRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.1); + this.put(EThreadPreset.LOW_IMPACT, getWorldGenDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.5); + this.put(EThreadPreset.AGGRESSIVE, 0.75); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); + public static int getBufferBuilderDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions bufferBuilders = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfBufferBuilderThreads, + private final ConfigEntryWithPresetOptions bufferBuilderThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfBufferBuilderThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -41,9 +53,20 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getBufferBuilderDefaultRunTimeRatio() { return 0.5; } + private final ConfigEntryWithPresetOptions bufferBuilderRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForBufferBuilderThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EThreadPreset.LOW_IMPACT, getBufferBuilderDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); + public static int getFileHandlerDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions fileHandlers = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, + private final ConfigEntryWithPresetOptions fileHandlerThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -52,9 +75,20 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getFileHandlerDefaultRunTimeRatio() { return 0.5; } + private final ConfigEntryWithPresetOptions fileHandlerRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EThreadPreset.LOW_IMPACT, getFileHandlerDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); + public static int getDataConverterDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions dataConverters = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads, + private final ConfigEntryWithPresetOptions dataConverterThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -63,17 +97,38 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getDataConverterDefaultRunTimeRatio() { return 0.25; } + private final ConfigEntryWithPresetOptions dataConverterRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForDataConverterThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.1); + this.put(EThreadPreset.LOW_IMPACT, getDataConverterDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); - public static int getChunkLodConvertersDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions chunkLodConverters = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads, + + public static int getChunkLodConverterDefaultThreadCount() { return getThreadCountByPercent(0.1); } + private final ConfigEntryWithPresetOptions chunkLodConverterThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); - this.put(EThreadPreset.LOW_IMPACT, getChunkLodConvertersDefaultThreadCount()); + this.put(EThreadPreset.LOW_IMPACT, getChunkLodConverterDefaultThreadCount()); this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.2)); this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getChunkLodConverterDefaultRunTimeRatio() { return 0.5; } + private final ConfigEntryWithPresetOptions chunkLodConverterRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForChunkLodConverterThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EThreadPreset.LOW_IMPACT, getDataConverterDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); @@ -85,16 +140,25 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan private ThreadPresetConfigEventHandler() { // add each config used by this preset - this.configList.add(this.worldGen); - this.configList.add(this.bufferBuilders); - this.configList.add(this.fileHandlers); - this.configList.add(this.dataConverters); - this.configList.add(this.chunkLodConverters); + this.configList.add(this.worldGenThreadCount); + this.configList.add(this.worldGenRunTime); + + this.configList.add(this.bufferBuilderThreadCount); + this.configList.add(this.bufferBuilderRunTime); + + this.configList.add(this.fileHandlerThreadCount); + this.configList.add(this.fileHandlerRunTime); + + this.configList.add(this.dataConverterThreadCount); + this.configList.add(this.dataConverterRunTime); + + this.configList.add(this.chunkLodConverterThreadCount); + this.configList.add(this.chunkLodConverterRunTime); for (ConfigEntryWithPresetOptions config : this.configList) { - // ignore try-using, the listener should only ever be added once and should never be removed + // ignore try-using, the listeners should only ever be added once and should never be removed new ConfigChangeListener<>(config.configEntry, (val) -> { this.onConfigValueChanged(); }); } } From a00e1c22c3058053793fae7e701275519fec7c89 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 20:56:34 -0500 Subject: [PATCH 07/10] Full Data source refactoring --- .../fullDatafile/FullDataFileHandler.java | 53 ++++++++----------- .../file/fullDatafile/FullDataMetaFile.java | 34 +++++++----- .../GeneratedFullDataFileHandler.java | 3 +- 3 files changed, 43 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 668a03cbf..81ec52f95 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -30,7 +30,6 @@ import java.util.function.Consumer; import java.util.function.Function; import static com.seibel.distanthorizons.core.util.FileScanUtil.LOD_FILE_POSTFIX; -import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX; public class FullDataFileHandler implements IFullDataSourceProvider { @@ -40,11 +39,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected static ExecutorService fileHandlerThreadPool; protected static ConfigChangeListener configListener; - + private final ConcurrentHashMap unloadedFiles = new ConcurrentHashMap<>(); private final ConcurrentHashMap fileBySectionPos = new ConcurrentHashMap<>(); public void ForEachFile(Consumer consumer) { this.fileBySectionPos.values().forEach(consumer); } - + + private LinkedList> onUpdatedListeners = new LinkedList<>(); + protected final IDhLevel level; protected final File saveDir; /** @@ -59,6 +60,11 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected final AtomicInteger topDetailLevel = new AtomicInteger(DhSectionPos.SECTION_REGION_DETAIL_LEVEL); protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; + + //=============// + // constructor // + //=============// + public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this.level = level; @@ -69,7 +75,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider } FileScanUtil.scanFiles(saveStructure, level.getLevelWrapper(), this, null); } - + + // constructor helpers // + /** * Caller must ensure that this method is called only once, * and that the {@link FullDataFileHandler} is not used before this method is called. @@ -179,6 +187,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider } } + + + protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile) { FullDataMetaFile metaFile = this.fileBySectionPos.get(pos); @@ -414,41 +425,19 @@ public class FullDataFileHandler implements IFullDataSourceProvider } return metaFile.flushAndSaveAsync(); } - - - private LinkedList> onUpdatedListeners = new LinkedList<>(); + + @Override public synchronized void addOnUpdatedListener(Consumer listener) { this.onUpdatedListeners.add(listener); } - -// @Override -// public long getCacheVersion(DhSectionPos sectionPos) -// { -// FullDataMetaFile file = this.files.get(sectionPos); -// if (file == null) -// { -// return 0; -// } -// return file.getCacheVersion(); -// } -// @Override -// public boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion) -// { -// FullDataMetaFile file = this.files.get(sectionPos); -// if (file == null) -// { -// return cacheVersion >= 0; -// } -// return file.isCacheVersionValid(cacheVersion); -// } - - protected IIncompleteFullDataSource makeDataSource(DhSectionPos pos) + protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { return pos.sectionDetailLevel <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL ? - HighDetailIncompleteFullDataSource.createEmpty(pos) : LowDetailIncompleteFullDataSource.createEmpty(pos); + HighDetailIncompleteFullDataSource.createEmpty(pos) : + LowDetailIncompleteFullDataSource.createEmpty(pos); } protected CompletableFuture sampleFromFiles(IIncompleteFullDataSource source, ArrayList existingFiles) @@ -486,7 +475,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider public CompletableFuture onCreateDataFile(FullDataMetaFile file) { DhSectionPos pos = file.pos; - IIncompleteFullDataSource source = this.makeDataSource(pos); + IIncompleteFullDataSource source = this.makeEmptyDataSource(pos); ArrayList existFiles = new ArrayList<>(); ArrayList missing = new ArrayList<>(); this.getDataFilesForPosition(pos, pos, existFiles, missing); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java index 8cd060da7..9ec036858 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java @@ -254,9 +254,9 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I }, executorService)); } - private void makeCreateCompletionStage(ExecutorService executorService, CompletableFuture completer) + private void makeCreateCompletionStage(CompletableFuture completer) { - makeUpdateCompletionStage(completer, this.fullDataSourceProvider.onCreateDataFile(this) + this.makeUpdateCompletionStage(completer, this.fullDataSourceProvider.onCreateDataFile(this) .thenApply((fullDataSource) -> { this.baseMetaData = this._makeBaseMetaData(fullDataSource); @@ -273,31 +273,39 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I CacheQueryResult result = this.getCachedDataSourceAsync(); - if (result.needsLoad) { - LodUtil.assertTrue(!inCrit); - inCrit = true; + if (result.needsLoad) + { + LodUtil.assertTrue(!this.inCrit); + this.inCrit = true; CompletableFuture future = result.future; // don't continue if the provider has been shut down ExecutorService executorService = this.fullDataSourceProvider.getIOExecutor(); if (executorService.isTerminated()) { - inCrit = false; - dataSourceLoadFutureRef.set(null); + this.inCrit = false; + this.dataSourceLoadFutureRef.set(null); future.complete(null); return future; } // create a new Meta file - if (!doesFileExist) { - makeCreateCompletionStage(executorService, future); + if (!this.doesFileExist) + { + this.makeCreateCompletionStage(future); } - // Otherwise, load and update file - else { - if (this.baseMetaData == null) throw new IllegalStateException("Meta data not loaded!"); - makeLoadCompletionStage(executorService, future); + else + { + // Otherwise, load and update file + if (this.baseMetaData == null) + { + throw new IllegalStateException("Meta data not loaded!"); + } + + this.makeLoadCompletionStage(executorService, future); } } + return result.future; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 6f9631bbf..dd315783b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -17,7 +17,6 @@ import com.seibel.distanthorizons.core.util.LodUtil; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; -import java.io.File; import java.lang.ref.WeakReference; import java.util.*; import java.util.concurrent.*; @@ -180,8 +179,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public CompletableFuture onCreateDataFile(FullDataMetaFile file) { DhSectionPos pos = file.pos; - IIncompleteFullDataSource data = makeDataSource(pos); CompletableFuture future = updateDataGenStatus(file, data); + IIncompleteFullDataSource data = makeEmptyDataSource(pos); // Cant start gen task, so return the data return future == null ? CompletableFuture.completedFuture(data) : future; } From f0d3ece345556aee1021be5ab23c054df482ff64 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 21:52:20 -0500 Subject: [PATCH 08/10] Full Data source refactoring 2 --- .../fullDatafile/FullDataFileHandler.java | 35 ++++++++------ .../GeneratedFullDataFileHandler.java | 46 +++++++++++-------- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 81ec52f95..010314c36 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -248,8 +248,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider return null; } - // This is a CAS with expected null value. this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + + // This is a CAS with expected null value. FullDataMetaFile metaFileCas = this.fileBySectionPos.putIfAbsent(pos, metaFile); return metaFileCas == null ? metaFile : metaFileCas; } @@ -266,6 +267,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider byte sectionDetail = posAreaToGet.sectionDetailLevel; boolean allEmpty = true; + // get all existing files for this position outerLoop: while (--sectionDetail >= this.minDetailLevel) { @@ -439,28 +441,33 @@ public class FullDataFileHandler implements IFullDataSourceProvider HighDetailIncompleteFullDataSource.createEmpty(pos) : LowDetailIncompleteFullDataSource.createEmpty(pos); } - - protected CompletableFuture sampleFromFiles(IIncompleteFullDataSource source, ArrayList existingFiles) + + /** populates the given data source using the given array of files */ + protected CompletableFuture sampleFromFileArray(IIncompleteFullDataSource recipientFullDataSource, ArrayList existingFiles) { // read in the existing data final ArrayList> loadDataFutures = new ArrayList<>(existingFiles.size()); for (FullDataMetaFile existingFile : existingFiles) { loadDataFutures.add(existingFile.loadOrGetCachedDataSourceAsync() - .exceptionally((ex) -> /*Ignore file read errors*/null) - .thenAccept((fullDataSource) -> + .exceptionally((ex) -> /*Ignore file read errors*/null) + .thenAccept((existingFullDataSource) -> + { + if (existingFullDataSource == null) { - if (fullDataSource == null) return; - //this.checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource); - //LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); - source.sampleFrom(fullDataSource); - }) + return; + } + + //LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); + recipientFullDataSource.sampleFrom(existingFullDataSource); + }) ); } - return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0])).thenApply(v -> source); + return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0])).thenApply(voidObj -> recipientFullDataSource); } - protected void makeFiles(ArrayList posList, ArrayList output) { + protected void makeFiles(ArrayList posList, ArrayList output) + { for (DhSectionPos missingPos : posList) { FullDataMetaFile newFile = this.getLoadOrMakeFile(missingPos, true); @@ -487,8 +494,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider } else { - makeFiles(missing, existFiles); - return sampleFromFiles(source, existFiles).thenApply(IIncompleteFullDataSource::tryPromotingToCompleteDataSource) + this.makeFiles(missing, existFiles); + return this.sampleFromFileArray(source, existFiles).thenApply(IIncompleteFullDataSource::tryPromotingToCompleteDataSource) .exceptionally((e) -> { FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, file, e); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index dd315783b..5e9c3d96e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -120,7 +120,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler ArrayList existingFiles = new ArrayList<>(); byte sectDetailLevel = (byte) (DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + maxSectDataDetailLevel); pos.forEachChildAtLevel(sectDetailLevel, p -> existingFiles.add(getLoadOrMakeFile(p, true))); - return sampleFromFiles(dataSource, existingFiles).thenApply(this::tryPromoteDataSource) + return sampleFromFileArray(dataSource, existingFiles).thenApply(this::tryPromoteDataSource) .exceptionally((e) -> { FullDataMetaFile newMetaFile = removeCorruptedFile(pos, file, e); @@ -152,26 +152,28 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // Try update the gen queue on this data source. If null, then nothing was done. @Nullable - private CompletableFuture updateDataGenStatus(FullDataMetaFile file, IIncompleteFullDataSource data) + private CompletableFuture updateFromExistingDataSources(FullDataMetaFile file, IIncompleteFullDataSource data) { DhSectionPos pos = file.pos; ArrayList existingFiles = new ArrayList<>(); ArrayList missingPositions = new ArrayList<>(); this.getDataFilesForPosition(pos, pos, existingFiles, missingPositions); - - if (missingPositions.size() == 1) { + + if (missingPositions.size() == 1) + { // Only missing myself. I.e. no child file data exists yet. - return tryStartGenTask(file, data); + return this.tryStartGenTask(file, data); } - else { - // Has stuff to sample. - makeFiles(missingPositions, existingFiles); - return sampleFromFiles(data, existingFiles).thenApply(this::tryPromoteDataSource) - .exceptionally((e) -> - { - FullDataMetaFile newMetaFile = removeCorruptedFile(pos, file, e); - return null; - }); + else + { + // There are other data source files to sample from. + this.makeFiles(missingPositions, existingFiles); + return this.sampleFromFileArray(data, existingFiles).thenApply(this::tryPromoteDataSource) + .exceptionally((e) -> + { + this.removeCorruptedFile(pos, file, e); + return null; + }); } } @@ -179,8 +181,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public CompletableFuture onCreateDataFile(FullDataMetaFile file) { DhSectionPos pos = file.pos; - CompletableFuture future = updateDataGenStatus(file, data); IIncompleteFullDataSource data = makeEmptyDataSource(pos); + CompletableFuture future = updateFromExistingDataSources(file, data); // Cant start gen task, so return the data return future == null ? CompletableFuture.completedFuture(data) : future; } @@ -205,12 +207,16 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } this.fireOnGenPosSuccessListeners(source.getSectionPos()); - if (source instanceof IIncompleteFullDataSource && !file.genQueueChecked) { + if (source instanceof IIncompleteFullDataSource && !file.genQueueChecked) + { WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null) { - CompletableFuture future = updateDataGenStatus(file, (IIncompleteFullDataSource) source); - if (future != null) { - return future.thenApply((newSource) -> { + if (worldGenQueue != null) + { + CompletableFuture future = this.updateFromExistingDataSources(file, (IIncompleteFullDataSource) source); + if (future != null) + { + return future.thenApply((newSource) -> + { onUpdated.accept(newSource); return newSource; }); From 6acb1a2e9f28a3de5064cc1e410c685bf6a1f515 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 21:56:05 -0500 Subject: [PATCH 09/10] Fix missing low detail LODs if the file didn't exist --- .../file/fullDatafile/FullDataFileHandler.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 010314c36..286e1ead4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -288,7 +288,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider continue; } - if (this.fileBySectionPos.containsKey(subPos)) + // check if a file for this pos exists, either loaded and unloaded + if (this.fileBySectionPos.containsKey(subPos) || this.unloadedFiles.containsKey(subPos)) { allEmpty = false; break outerLoop; @@ -317,6 +318,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider DhSectionPos childPos = pos.getChildByIndex(childIndex); if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos)) { + // load the file if it isn't already + if (this.unloadedFiles.containsKey(childPos)) + { + this.getLoadOrMakeFile(childPos, true); + } + + FullDataMetaFile metaFile = this.fileBySectionPos.get(childPos); if (metaFile != null) { @@ -386,12 +394,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData) { FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos); -// if (metaFile == null && sectionPos.sectionDetailLevel <= this.topDetailLevel.get()) -// { -// // create a new file if one doesn't exist, -// // this is done so we don't end up with holes where LODs should have been generated -// metaFile = this.getLoadOrMakeFile(sectionPos, true); -// } if (metaFile != null) { // there is a file for this position From 9d518a5fd662392feeb4379917c21bdd97d638bc Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 22:07:59 -0500 Subject: [PATCH 10/10] fully remove band-aid fix for missing low detail LODs --- .../core/file/fullDatafile/FullDataFileHandler.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 286e1ead4..b1ddbda49 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -48,16 +48,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected final IDhLevel level; protected final File saveDir; - /** - * The starting value here denotes how far into the tree LOD writes should occur.
- * This is a band-aid fix to prevent lower detail sections from not being generated until the detail level - * is requested.

- * - * Note: that problem may still happen at a sufficiently large render distance, - * however this should kick the problem down the road - * far enough that it can be ignored for now. - */ - protected final AtomicInteger topDetailLevel = new AtomicInteger(DhSectionPos.SECTION_REGION_DETAIL_LEVEL); + protected final AtomicInteger topDetailLevel = new AtomicInteger(0); protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET;