From 51bb3eec3d51f1e26bd498c773628b7bc6bf992c Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Sun, 16 Jul 2023 17:23:31 +0800 Subject: [PATCH 01/20] Fix f3 msg causing mem leaks & render file throwing (harmless) exceptions on shutdown --- .../file/fullDatafile/FullDataMetaFile.java | 10 +---- .../file/renderfile/RenderMetaDataFile.java | 7 +++- .../renderfile/RenderSourceFileHandler.java | 2 +- .../core/generation/WorldGenerationQueue.java | 8 +--- .../core/level/ClientLevelModule.java | 4 +- .../core/logging/f3/F3Screen.java | 38 ++++++++++--------- .../distanthorizons/core/util/LodUtil.java | 14 ++++++- .../UncheckedInterruptedException.java | 8 ++-- .../core/world/DhClientServerWorld.java | 4 +- 9 files changed, 55 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java index 150acb499..8cd060da7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java @@ -205,16 +205,10 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I }) .whenComplete((fullDataSource, ex) -> { - if (ex instanceof CompletionException) { - ex = ex.getCause(); - } - if (ex instanceof InterruptedException || ex instanceof RejectedExecutionException) - { - // this exception can be ignored - } - else if (ex != null) { + if (ex != null && !LodUtil.isInterruptOrReject(ex)) { LOGGER.error("Error updating file "+this.file+": ", ex); } + if (fullDataSource != null) { new DataObjTracker(fullDataSource); new DataObjSoftTracker(this, fullDataSource); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java index 16b5b3641..4d8d9dce4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java @@ -14,6 +14,7 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.util.AtomicsUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -222,7 +223,8 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements { if (ex != null) { - LOGGER.error("Uncaught error on creation {}: ", this.file, ex); + if (!LodUtil.isInterruptOrReject(ex)) + LOGGER.error("Uncaught error on creation {}: ", this.file, ex); cachedRenderDataSource = new SoftReference<>(null); renderSourceLoadFutureRef.set(null); future.complete(null); @@ -263,7 +265,8 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements { if (ex != null) { - LOGGER.error("Error loading file {}: ", this.file, ex); + if (!LodUtil.isInterruptOrReject(ex)) + LOGGER.error("Error loading file {}: ", this.file, ex); cachedRenderDataSource = new SoftReference<>(null); renderSourceLoadFutureRef.set(null); future.complete(null); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 43b6e7fac..fd3fd431b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -418,7 +418,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider int ignoreEmptyWarning = 0; } - else if (!UncheckedInterruptedException.isThrowableInterruption(ex)) + else if (!UncheckedInterruptedException.isInterrupt(ex)) { LOGGER.error("Exception when updating render file using data source: ", ex); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 11f79963f..0e003dd77 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -14,13 +14,9 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandler; -import com.seibel.distanthorizons.core.generation.tasks.*; -import com.seibel.distanthorizons.core.pos.*; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.util.ThreadUtil; -import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; -import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree; import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; @@ -430,7 +426,7 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable if (exception != null) { // don't log the shutdown exceptions - if (!UncheckedInterruptedException.isThrowableInterruption(exception) + if (!UncheckedInterruptedException.isInterrupt(exception) && !(exception instanceof CancellationException || exception.getCause() instanceof CancellationException)) { LOGGER.error("Error generating data for section "+taskPos, exception); @@ -566,7 +562,7 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable exception = exception.getCause(); } - if (!UncheckedInterruptedException.isThrowableInterruption(exception) && !(exception instanceof CancellationException)) + if (!UncheckedInterruptedException.isInterrupt(exception) && !(exception instanceof CancellationException)) { LOGGER.error("Error when terminating data generation for section "+runningTaskGroup.group.pos, exception); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index c4f84e7cb..ba0499ccd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -23,10 +23,11 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.coreapi.util.math.Mat4f; import org.apache.logging.log4j.Logger; +import java.io.Closeable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; -public class ClientLevelModule { +public class ClientLevelModule implements Closeable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); private final IDhClientLevel parent; @@ -192,6 +193,7 @@ public class ClientLevelModule { ClientRenderState.close(); } } + f3Message.close(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java b/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java index f17d99f72..f5ab0e804 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java @@ -3,10 +3,7 @@ package com.seibel.distanthorizons.core.logging.f3; import com.seibel.distanthorizons.coreapi.ModInfo; import java.io.Closeable; -import java.util.Arrays; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.function.Supplier; public class F3Screen @@ -15,22 +12,25 @@ public class F3Screen "", // blank line for spacing ModInfo.READABLE_NAME + " version: " + ModInfo.VERSION }; - private static final LinkedList SELF_UPDATE_MESSAGE_LIST = new LinkedList<>(); + private static final List SELF_UPDATE_MESSAGE_LIST = Collections.synchronizedList(new LinkedList<>()); public static void addStringToDisplay(List list) { list.addAll(Arrays.asList(DEFAULT_STRING)); - Iterator iterator = SELF_UPDATE_MESSAGE_LIST.iterator(); - while (iterator.hasNext()) - { - Message message = iterator.next(); - if (message == null) + synchronized (SELF_UPDATE_MESSAGE_LIST) + { + Iterator iterator = SELF_UPDATE_MESSAGE_LIST.iterator(); + while (iterator.hasNext()) { - iterator.remove(); - } - else - { - message.printTo(list); + Message message = iterator.next(); + if (message == null) + { + iterator.remove(); + } + else + { + message.printTo(list); + } } } } @@ -45,12 +45,16 @@ public class F3Screen // and because this class shouldn't be used in a try {} block. public static abstract class Message implements Closeable { - protected Message() { SELF_UPDATE_MESSAGE_LIST.add(this); } + protected Message() { + SELF_UPDATE_MESSAGE_LIST.add(this); + } public abstract void printTo(List output); @Override - public void close() { SELF_UPDATE_MESSAGE_LIST.remove(this); } + public void close() { + boolean removed = SELF_UPDATE_MESSAGE_LIST.remove(this); + } } public static class StaticMessage extends Message diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java index 56885f1ab..02bad3b97 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java @@ -20,6 +20,8 @@ package com.seibel.distanthorizons.core.util; import java.util.Iterator; +import java.util.concurrent.CompletionException; +import java.util.concurrent.RejectedExecutionException; import com.seibel.distanthorizons.api.enums.config.EVanillaOverdraw; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; @@ -29,6 +31,7 @@ import com.seibel.distanthorizons.core.pos.Pos2D; import com.seibel.distanthorizons.core.render.vertexFormat.DefaultLodVertexFormats; import com.seibel.distanthorizons.core.render.vertexFormat.LodVertexFormat; import com.seibel.distanthorizons.core.util.gridList.EdgeDistanceBooleanGrid; +import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftRenderWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IDimensionTypeWrapper; @@ -297,6 +300,15 @@ public class LodUtil public static void assertToDo() { throw new AssertFailureException("TODO!"); } - + + public static Throwable ensureUnwrap(Throwable t) { + return t instanceof CompletionException ? ensureUnwrap(t.getCause()) : t; + } + + public static boolean isInterruptOrReject(Throwable t) { + Throwable unwrapped = LodUtil.ensureUnwrap(t); + return UncheckedInterruptedException.isInterrupt(unwrapped) || + unwrapped instanceof RejectedExecutionException; + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/UncheckedInterruptedException.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/UncheckedInterruptedException.java index c385e88fa..f88ad376a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/UncheckedInterruptedException.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/UncheckedInterruptedException.java @@ -1,5 +1,7 @@ package com.seibel.distanthorizons.core.util.objects; +import com.seibel.distanthorizons.core.util.LodUtil; + import java.util.concurrent.CompletionException; public class UncheckedInterruptedException extends RuntimeException { @@ -38,8 +40,8 @@ public class UncheckedInterruptedException extends RuntimeException { rethrowIfIsInterruption(t.getCause()); } } - public static boolean isThrowableInterruption(Throwable t) { - return t instanceof InterruptedException || t instanceof UncheckedInterruptedException - || (t instanceof CompletionException && isThrowableInterruption(t.getCause())); + public static boolean isInterrupt(Throwable t) { + Throwable unwrapped = LodUtil.ensureUnwrap(t); + return unwrapped instanceof InterruptedException || unwrapped instanceof UncheckedInterruptedException; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java index 728a82cb4..6c38ee926 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java @@ -101,7 +101,9 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor } else { - // TODO why is this called here? + // If the level wrapper is a Client Level Wrapper, then that means the client side leaves the level, + // but note that the server side still has the level loaded. So, we don't want to unload the level, + // we just want to stop rendering it. this.levelObjMap.remove(wrapper).stopRenderer(); // Ignore resource warning. The level obj is referenced elsewhere. } } From c5041de5d4e19cee4423bf85ceeb765b06ab8ff8 Mon Sep 17 00:00:00 2001 From: coolGi Date: Sun, 16 Jul 2023 23:07:14 +0930 Subject: [PATCH 02/20] Added proper clamping so values don't get out of range --- .../distanthorizons/core/config/Config.java | 4 +- .../core/config/NumberUtil.java | 38 +++++++++ .../core/config/file/ConfigFileHandling.java | 10 +-- .../core/config/types/ConfigEntry.java | 78 +++++++++++++++---- 4 files changed, 108 insertions(+), 22 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index c3c27afc0..d76f23d19 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -51,7 +51,7 @@ import java.util.*; * Otherwise, you will have issues where only some of the config entries will exist when your listener is created. * * @author coolGi - * @version 2023-6-12 + * @version 2023-7-16 */ public class Config @@ -456,7 +456,7 @@ public class Config .build(); public static ConfigEntry noiseSteps = new ConfigEntry.Builder() - .setMinDefaultMax(0, 4, null) + .setMinDefaultMax(1, 4, null) .comment("" + "How many steps of noise should be applied to LODs?") .build(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java new file mode 100644 index 000000000..9d63ce909 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java @@ -0,0 +1,38 @@ +package com.seibel.distanthorizons.core.config; + +import java.util.HashMap; +import java.util.Map; + +/** + * Gets the minimum or maximum values of a given type + * + * @author coolGi + * @version 2023-7-16 + */ +// TODO: Should this be moved out of here into somewhere like the util section +public class NumberUtil { + // Is there no better way of doing this? + public static Map minValues = new HashMap() {{ + put(Byte.class, Byte.MIN_VALUE); + put(Short.class, Short.MIN_VALUE); + put(Integer.class, Integer.MIN_VALUE); + put(Long.class, Long.MIN_VALUE); + put(Double.class, Double.MIN_VALUE); + put(Float.class, Float.MIN_VALUE); + }}; + public static Map maxValues = new HashMap() {{ + put(Byte.class, Byte.MAX_VALUE); + put(Short.class, Short.MAX_VALUE); + put(Integer.class, Integer.MAX_VALUE); + put(Long.class, Long.MAX_VALUE); + put(Double.class, Double.MAX_VALUE); + put(Float.class, Float.MAX_VALUE); + }}; + + public static Number getMinimum(Class c) { + return minValues.get(c); + } + public static Number getMaximum(Class c) { + return maxValues.get(c); + } +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java b/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java index 2f51a9571..0cb1aaa2a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java @@ -17,7 +17,7 @@ import java.nio.file.Path; * Handles reading and writing config files. * * @author coolGi - * @version 2022-9-9 + * @version 2023-7-16 */ public class ConfigFileHandling { private static final Logger LOGGER = ConfigBase.LOGGER; @@ -136,7 +136,7 @@ public class ConfigFileHandling { if (workConfig.contains(entry.getNameWCategory())) { try { if (entry.getType().isEnum()) { - entry.setWithoutSaving((T) ( workConfig.getEnum(entry.getNameWCategory(), (Class) entry.getType()) )); + entry.setWithoutSaving((T) ( workConfig.getEnum(entry.getNameWCategory(), (Class) entry.getType()))); return; } if (ConfigTypeConverters.convertObjects.containsKey(entry.getType())) { @@ -146,11 +146,7 @@ public class ConfigFileHandling { if (entry.getType() == workConfig.get(entry.getNameWCategory()).getClass()) { // If the types are the same entry.setWithoutSaving((T) workConfig.get(entry.getNameWCategory())); - - if (entry.isValid() == 0) return; - else if (entry.isValid() == -1) entry.setWithoutSaving(entry.getMin()); - else if (entry.isValid() == 1) entry.setWithoutSaving(entry.getMax()); - return; + entry.clampWithinRange(); } LOGGER.warn("Entry ["+ entry.getNameWCategory() +"] is invalid. Expected " + entry.getType() + " but got " + workConfig.get(entry.getNameWCategory()).getClass() + ". Using default value."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java index 6aa4b9d28..56f2953cb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java @@ -1,6 +1,7 @@ package com.seibel.distanthorizons.core.config.types; +import com.seibel.distanthorizons.core.config.NumberUtil; import com.seibel.distanthorizons.core.config.listeners.IConfigListener; import com.seibel.distanthorizons.core.config.types.enums.EConfigEntryAppearance; import com.seibel.distanthorizons.core.config.types.enums.EConfigEntryPerformance; @@ -14,7 +15,7 @@ import java.util.Arrays; * for types that are not supported by it look in ConfigBase * * @author coolGi - * @version 2022-5-26 + * @version 2023-7-16 */ public class ConfigEntry extends AbstractConfigType> implements IConfigEntry { @@ -41,8 +42,7 @@ public class ConfigEntry extends AbstractConfigType> implem super(appearance, value); this.defaultValue = value; this.comment = comment; - this.min = min; - this.max = max; + this.setMinMax(min, max); this.allowApiOverride = allowApiOverride; this.performance = performance; this.listenerList = listenerList; @@ -104,19 +104,47 @@ public class ConfigEntry extends AbstractConfigType> implem public T getMin() { return this.min; } /** Sets the min value */ @Override - public void setMin(T newMin) { this.min = newMin; } + @SuppressWarnings("unchecked") // Suppress due to its always safe + public void setMin(T newMin) { + if (newMin == null) + newMin = (T) NumberUtil.getMinimum(newMin.getClass()); + this.min = newMin; + } /** Gets the max value */ @Override public T getMax() { return this.max; } /** Sets the max value */ @Override - public void setMax(T newMax) { this.max = newMax; } - /** Sets the min and max in 1 setter */ + @SuppressWarnings("unchecked") // Suppress due to its always safe + public void setMax(T newMax) { + if (newMax == null) + newMax = (T) NumberUtil.getMinimum(newMax.getClass()); + this.max = newMax; + } + /** Sets the min and max within a single setter */ @Override - public void setMinMax(T newMin, T newMax) - { - this.max = newMin; - this.min = newMax; + public void setMinMax(T newMin, T newMax) { + this.setMin(newMin); + this.setMax(newMax); + } + + /** + * Clamps the value within the set range + * @apiNote This does not save the value + */ + public void clampWithinRange() { this.clampWithinRange(this.min, this.max); } + /** + * Clamps the value within a set range + * + * @param min The minimum that the value can be + * @param max The maximum that the value can be + * @apiNote This does not save the value + */ + @SuppressWarnings("unchecked") // Suppress due to its always safe + public void clampWithinRange(T min, T max) { + byte validness = this.isValid(min, max); + if (validness == -1) this.value = (T) NumberUtil.getMinimum(min.getClass()); + if (validness == 1) this.value = (T) NumberUtil.getMinimum(max.getClass()); } @Override @@ -150,17 +178,41 @@ public class ConfigEntry extends AbstractConfigType> implem *

-1 == number too low */ @Override - public byte isValid() { return isValid(this.value); } + public byte isValid() { return isValid(this.value, this.min, this.max); } /** * Checks if a new value is valid * + * @param value Value that is being checked whether valid * @return 0 == valid *

2 == invalid *

1 == number too high *

-1 == number too low */ - @Override - public byte isValid(T value) { + @Override + public byte isValid(T value) { return this.isValid(value, this.min, this.max); } + /** + * Checks if a new value is valid + * + * @param min The minimum that the value can be + * @param max The maximum that the value can be + * @return 0 == valid + *

2 == invalid + *

1 == number too high + *

-1 == number too low + */ + public byte isValid(T min, T max) { return this.isValid(this.value, min, max); } + /** + * Checks if a new value is valid + * + * @param value Value that is being checked whether valid + * @param min The minimum that the value can be + * @param max The maximum that the value can be + * @return 0 == valid + *

2 == invalid + *

1 == number too high + *

-1 == number too low + */ + public byte isValid(T value, T min, T max) { if (this.configBase.disableMinMax) return 0; From 8727cd09af82cfeedea963c3557edb3482e9ea68 Mon Sep 17 00:00:00 2001 From: coolGi Date: Sun, 16 Jul 2023 23:40:35 +0930 Subject: [PATCH 03/20] Allowd option for values to go out of range --- .../distanthorizons/core/config/Config.java | 12 ++++++++++-- .../core/config/ConfigBase.java | 2 +- .../UnsafeValuesConfigListener.java | 19 +++++++++++++++++++ .../core/config/file/ConfigFileHandling.java | 1 + .../core/config/types/ConfigEntry.java | 10 +++++----- 5 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/UnsafeValuesConfigListener.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index d76f23d19..05f70c537 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.api.enums.config.quickOptions.EThreadPreset; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiDistantGeneratorMode; import com.seibel.distanthorizons.core.config.eventHandlers.QuickRenderToggleConfigEventHandler; import com.seibel.distanthorizons.core.config.eventHandlers.RenderCacheConfigEventHandler; +import com.seibel.distanthorizons.core.config.eventHandlers.UnsafeValuesConfigListener; import com.seibel.distanthorizons.core.config.eventHandlers.presets.ThreadPresetConfigEventHandler; import com.seibel.distanthorizons.core.config.eventHandlers.presets.RenderQualityPresetConfigEventHandler; import com.seibel.distanthorizons.core.config.types.ConfigCategory; @@ -1011,10 +1012,17 @@ public class Config + " Additionally, only stuff that's loaded after you enable this \n" + " will render their debug wireframes.") .build(); - + + // Note: This will reset on game restart, and should have a warning on the tooltip + public static ConfigEntry allowUnsafeValues = new ConfigEntry.Builder() + .set(false) + .setAppearance(EConfigEntryAppearance.ONLY_IN_GUI) + .addListener(UnsafeValuesConfigListener.INSTANCE) + .build(); + // can be set to public inorder to show in the config file and UI - private static ConfigCategory exampleConfigScreen = new ConfigCategory.Builder() + public static ConfigCategory exampleConfigScreen = new ConfigCategory.Builder() .set(ExampleConfigScreen.class) .build(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java index 580f92143..7b0ddd4e3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java @@ -68,7 +68,7 @@ public class ConfigBase }}; /** Disables the minimum and maximum of any variable */ - public boolean disableMinMax = false; // Very fun to use + public boolean disableMinMax = false; // Very fun to use, but should always be disabled by default public final List> entries = new ArrayList<>(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/UnsafeValuesConfigListener.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/UnsafeValuesConfigListener.java new file mode 100644 index 000000000..4c0eca3bc --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/UnsafeValuesConfigListener.java @@ -0,0 +1,19 @@ +package com.seibel.distanthorizons.core.config.eventHandlers; + +import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.config.listeners.IConfigListener; + +public class UnsafeValuesConfigListener implements IConfigListener { + public static UnsafeValuesConfigListener INSTANCE = new UnsafeValuesConfigListener(); + + @Override + public void onConfigValueSet() { + Config.Client.Advanced.Debugging.allowUnsafeValues.configBase.disableMinMax = + Config.Client.Advanced.Debugging.allowUnsafeValues.get(); + } + + @Override + public void onUiModify() { + + } +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java b/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java index 0cb1aaa2a..270ecda4c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandling.java @@ -147,6 +147,7 @@ public class ConfigFileHandling { if (entry.getType() == workConfig.get(entry.getNameWCategory()).getClass()) { // If the types are the same entry.setWithoutSaving((T) workConfig.get(entry.getNameWCategory())); entry.clampWithinRange(); + return; } LOGGER.warn("Entry ["+ entry.getNameWCategory() +"] is invalid. Expected " + entry.getType() + " but got " + workConfig.get(entry.getNameWCategory()).getClass() + ". Using default value."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java index 56f2953cb..76112607b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java @@ -107,7 +107,7 @@ public class ConfigEntry extends AbstractConfigType> implem @SuppressWarnings("unchecked") // Suppress due to its always safe public void setMin(T newMin) { if (newMin == null) - newMin = (T) NumberUtil.getMinimum(newMin.getClass()); + newMin = (T) NumberUtil.getMinimum(this.value.getClass()); this.min = newMin; } /** Gets the max value */ @@ -118,7 +118,7 @@ public class ConfigEntry extends AbstractConfigType> implem @SuppressWarnings("unchecked") // Suppress due to its always safe public void setMax(T newMax) { if (newMax == null) - newMax = (T) NumberUtil.getMinimum(newMax.getClass()); + newMax = (T) NumberUtil.getMinimum(this.value.getClass()); this.max = newMax; } /** Sets the min and max within a single setter */ @@ -143,8 +143,8 @@ public class ConfigEntry extends AbstractConfigType> implem @SuppressWarnings("unchecked") // Suppress due to its always safe public void clampWithinRange(T min, T max) { byte validness = this.isValid(min, max); - if (validness == -1) this.value = (T) NumberUtil.getMinimum(min.getClass()); - if (validness == 1) this.value = (T) NumberUtil.getMinimum(max.getClass()); + if (validness == -1) this.value = (T) NumberUtil.getMinimum(this.value.getClass()); + if (validness == 1) this.value = (T) NumberUtil.getMaximum(this.value.getClass()); } @Override @@ -221,7 +221,7 @@ public class ConfigEntry extends AbstractConfigType> implem if (Number.class.isAssignableFrom(value.getClass())) { // Only check min max if it is a number if (this.max != null && Float.parseFloat(value.toString()) > Float.parseFloat(max.toString())) return 1; - if (this.min != null && Float.parseFloat(value.toString()) < Float.parseFloat(min.toString())) + if (min != null && Float.parseFloat(value.toString()) < Float.parseFloat(min.toString())) return -1; return 0; From 76b464d5d76381e8c9ac771341566010872c691e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 09:25:44 -0500 Subject: [PATCH 04/20] Revert LOD hole bandaid fix due to performance issues reverts main change in 4a069b42d8eb0d99debe3c5a832d2bc3c8715e62 --- .../core/file/fullDatafile/FullDataFileHandler.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 8b9717c8f..66d5fcd4c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -373,12 +373,12 @@ public class FullDataFileHandler implements IFullDataSourceProvider private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData) { FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos); - if (metaFile == null && sectionPos.sectionDetailLevel <= this.topDetailLevel.get()) - { - // create a new file if one doesn't exist, - // this is done so we don't end up with holes where LODs should have been generated - metaFile = this.getLoadOrMakeFile(sectionPos, true); - } +// if (metaFile == null && sectionPos.sectionDetailLevel <= this.topDetailLevel.get()) +// { +// // create a new file if one doesn't exist, +// // this is done so we don't end up with holes where LODs should have been generated +// metaFile = this.getLoadOrMakeFile(sectionPos, true); +// } if (metaFile != null) { // there is a file for this position From ce3a06e4102fda2806e36b40297e2c951db6641e Mon Sep 17 00:00:00 2001 From: coolGi Date: Sun, 16 Jul 2023 23:57:27 +0930 Subject: [PATCH 05/20] fix to commit c5041de5 --- .../core/config/ConfigBase.java | 7 +--- .../core/config/NumberUtil.java | 32 ++++++++++++++++++- .../core/config/types/ConfigEntry.java | 21 ++++-------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java index 7b0ddd4e3..9d8e601a9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java @@ -52,12 +52,7 @@ public class ConfigBase */ public static final List> acceptableInputs = new ArrayList>() {{ add(Boolean.class); - add(Byte.class); - add(Integer.class); - add(Double.class); - add(Short.class); - add(Long.class); - add(Float.class); + add(Number.class); // Contains: Byte, Short, Int, Long, Double, Float0 add(String.class); // TODO[CONFIG]: Check the type of these is valid diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java index 9d63ce909..1731da4d0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/NumberUtil.java @@ -4,7 +4,7 @@ import java.util.HashMap; import java.util.Map; /** - * Gets the minimum or maximum values of a given type + * Helps with working with numbers that the value of which is unknown * * @author coolGi * @version 2023-7-16 @@ -35,4 +35,34 @@ public class NumberUtil { public static Number getMaximum(Class c) { return maxValues.get(c); } + + /** Does a greater than (>) operator on any number */ + public static boolean greaterThan(Number a, Number b) { + if (a.getClass() != b.getClass()) + return false; + Class typeClass = a.getClass(); + + if (typeClass == Byte.class) return a.byteValue() > b.byteValue(); + if (typeClass == Short.class) return a.shortValue() > b.shortValue(); + if (typeClass == Integer.class) return a.intValue() > b.intValue(); + if (typeClass == Long.class) return a.longValue() > b.longValue(); + if (typeClass == Double.class) return a.doubleValue() > b.doubleValue(); + if (typeClass == Float.class) return a.floatValue() > b.floatValue(); + return false; + } + + /** Does a less than (<) operator on any number */ + public static boolean lessThan(Number a, Number b) { + if (a.getClass() != b.getClass()) + return false; + Class typeClass = a.getClass(); + + if (typeClass == Byte.class) return a.byteValue() < b.byteValue(); + if (typeClass == Short.class) return a.shortValue() < b.shortValue(); + if (typeClass == Integer.class) return a.intValue() < b.intValue(); + if (typeClass == Long.class) return a.longValue() < b.longValue(); + if (typeClass == Double.class) return a.doubleValue() < b.doubleValue(); + if (typeClass == Float.class) return a.floatValue() < b.floatValue(); + return false; + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java index 76112607b..a297bdb3e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java @@ -42,7 +42,8 @@ public class ConfigEntry extends AbstractConfigType> implem super(appearance, value); this.defaultValue = value; this.comment = comment; - this.setMinMax(min, max); + this.min = min; + this.max = max; this.allowApiOverride = allowApiOverride; this.performance = performance; this.listenerList = listenerList; @@ -104,23 +105,13 @@ public class ConfigEntry extends AbstractConfigType> implem public T getMin() { return this.min; } /** Sets the min value */ @Override - @SuppressWarnings("unchecked") // Suppress due to its always safe - public void setMin(T newMin) { - if (newMin == null) - newMin = (T) NumberUtil.getMinimum(this.value.getClass()); - this.min = newMin; - } + public void setMin(T newMin) { this.min = newMin; } /** Gets the max value */ @Override public T getMax() { return this.max; } /** Sets the max value */ @Override - @SuppressWarnings("unchecked") // Suppress due to its always safe - public void setMax(T newMax) { - if (newMax == null) - newMax = (T) NumberUtil.getMinimum(this.value.getClass()); - this.max = newMax; - } + public void setMax(T newMax) { this.max = newMax; } /** Sets the min and max within a single setter */ @Override public void setMinMax(T newMin, T newMax) { @@ -219,9 +210,9 @@ public class ConfigEntry extends AbstractConfigType> implem if (value.getClass() != this.value.getClass()) // If the 2 variables aren't the same type then it will be invalid return 2; if (Number.class.isAssignableFrom(value.getClass())) { // Only check min max if it is a number - if (this.max != null && Float.parseFloat(value.toString()) > Float.parseFloat(max.toString())) + if (max != null && NumberUtil.greaterThan((Number) value, (Number) max)) return 1; - if (min != null && Float.parseFloat(value.toString()) < Float.parseFloat(min.toString())) + if (min != null && NumberUtil.lessThan((Number) value, (Number) min)) return -1; return 0; From 66f4595b7b22b25a30046e96c7befb7edeae83aa Mon Sep 17 00:00:00 2001 From: coolGi Date: Mon, 17 Jul 2023 00:27:32 +0930 Subject: [PATCH 06/20] Fixed values being able to go over number limit --- .../com/seibel/distanthorizons/core/config/ConfigBase.java | 7 ++++++- .../distanthorizons/core/config/types/ConfigEntry.java | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java index 9d8e601a9..7b0ddd4e3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigBase.java @@ -52,7 +52,12 @@ public class ConfigBase */ public static final List> acceptableInputs = new ArrayList>() {{ add(Boolean.class); - add(Number.class); // Contains: Byte, Short, Int, Long, Double, Float0 + add(Byte.class); + add(Integer.class); + add(Double.class); + add(Short.class); + add(Long.class); + add(Float.class); add(String.class); // TODO[CONFIG]: Check the type of these is valid diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java index a297bdb3e..6cc64d209 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/types/ConfigEntry.java @@ -207,7 +207,7 @@ public class ConfigEntry extends AbstractConfigType> implem if (this.configBase.disableMinMax) return 0; - if (value.getClass() != this.value.getClass()) // If the 2 variables aren't the same type then it will be invalid + if (value == null || this.value == null || value.getClass() != this.value.getClass()) // If the 2 variables aren't the same type then it will be invalid return 2; if (Number.class.isAssignableFrom(value.getClass())) { // Only check min max if it is a number if (max != null && NumberUtil.greaterThan((Number) value, (Number) max)) From 24b32ad6c980b3349bc1ed0eacbcf5b65831483d Mon Sep 17 00:00:00 2001 From: coolGi Date: Mon, 17 Jul 2023 01:32:38 +0930 Subject: [PATCH 07/20] Slight changes to the api fabric.mod.json to make it look like a library in modmenu --- api/src/main/resources/fabric.mod.json | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/src/main/resources/fabric.mod.json b/api/src/main/resources/fabric.mod.json index 263109227..01c9a7321 100644 --- a/api/src/main/resources/fabric.mod.json +++ b/api/src/main/resources/fabric.mod.json @@ -4,7 +4,15 @@ "version": "1.0", "name": "DhApi", - "description": "This is just a placeholder to make uploading to Modrinth happy. Don't try running this jar in Minecraft. \nNote: while the API mod appears in mod menu lists when running DH via gradle, it won't appear when DH has been built and run in retail MC." - + "description": "This is just a placeholder to make uploading to Modrinth happy. Don't try running this jar in Minecraft. \nNote: while the API mod appears in mod menu lists when running DH via gradle, it won't appear when DH has been built and run in retail MC.", + + "custom": { + "modmenu": { + "badges": ["library"], + "parent": { + "id": "distanthorizons" + } + } + } } From 2444d7068f8da961f9828a1efb3fdb86c81885b0 Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Mon, 17 Jul 2023 00:05:51 +0800 Subject: [PATCH 08/20] Tried and failed to fix a bug. But hey, better loading time...? --- .../GeneratedFullDataFileHandler.java | 5 +- .../file/renderfile/RenderMetaDataFile.java | 17 ++++-- .../renderfile/RenderSourceFileHandler.java | 60 ++++++++----------- .../core/generation/BatchGenerator.java | 10 +++- .../core/generation/WorldGenerationQueue.java | 3 +- .../distanthorizons/core/util/LodUtil.java | 4 +- 6 files changed, 51 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index eb5e699fb..6f9631bbf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -47,10 +47,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler @Override public CompletableFuture read(DhSectionPos pos) { - return super.read(pos).whenComplete((fullDataSource, ex) -> - { - //this.checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource); - }); + return super.read(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java index 4d8d9dce4..645c55678 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java @@ -139,7 +139,9 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements { return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist. } - CompletableFuture source = getCachedDataSourceAsync(true); + // FIXME: TODO: Change doTriggerUpdate to true. Currently is false cause a dead future making render handler hang, + // and that render cache aren't actually used really yet due to missing versioning atm. So disabling for now. + CompletableFuture source = getCachedDataSourceAsync(false); if (source == null) { return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save. @@ -187,6 +189,9 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements this.fileHandler.onReadRenderSourceLoadedFromCacheAsync(this, cachedRenderDataSource) // wait for the handler to finish before returning the renderSource .handle((voidObj, ex) -> { + if (ex != null) { + LOGGER.error("Error while updating render source from cache", ex); + } newFuture.complete(cachedRenderDataSource); renderSourceLoadFutureRef.set(null); return null; @@ -218,7 +223,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements this.baseMetaData = this.makeMetaData(renderSource); return renderSource; }) - .thenApply((renderSource) -> this.fileHandler.onRenderFileLoaded(renderSource, this)) + .thenCompose((renderSource) -> this.fileHandler.onRenderFileLoaded(renderSource, this)) .whenComplete((renderSource, ex) -> { if (ex != null) @@ -239,13 +244,13 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements } else { - CompletableFuture.supplyAsync(() -> + CompletableFuture.supplyAsync(() -> { if (this.baseMetaData == null) { throw new IllegalStateException("Meta data not loaded!"); } - + // Load the file. ColumnRenderSource renderSource; try (FileInputStream fileInputStream = this.getFileInputStream(); @@ -257,10 +262,10 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements { throw new CompletionException(ex); } - - renderSource = this.fileHandler.onRenderFileLoaded(renderSource, this); return renderSource; }, fileReaderThreads) + // TODO: Check for file version and only update if needed. + .thenCompose((renderSource) -> this.fileHandler.onRenderFileLoaded(renderSource, this)) .whenComplete((renderSource, ex) -> { if (ex != null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index fd3fd431b..adfcf7c55 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -14,6 +14,7 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.util.FileScanUtil; import com.seibel.distanthorizons.core.util.FileUtil; +import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.config.Config; @@ -378,9 +379,9 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider DebugRenderer.BoxWithLife box = new DebugRenderer.BoxWithLife(new DebugRenderer.Box(renderSource.sectionPos, 74f, 86f, 0.1f, Color.red), 1.0, 32f, Color.green.darker()); // get the full data source loading future - CompletableFuture fullDataSourceFuture = this.fullDataSourceProvider.read(renderSource.getSectionPos()); - fullDataSourceFuture = fullDataSourceFuture.thenApply((fullDataSource) -> - { + CompletableFuture fullDataSourceFuture = + this.fullDataSourceProvider.read(renderSource.getSectionPos()) + .thenApply((fullDataSource) -> { // the fullDataSource can be null if the thread this was running on was interrupted box.box.color = Color.yellow.darker(); return fullDataSource; @@ -400,40 +401,30 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider { if (ex == null) { - this.writeRenderSourceToFile(renderSource, file, newRenderSource); + try { + this.writeRenderSourceToFile(renderSource, file, newRenderSource); + } catch (Throwable e) { + LOGGER.error("Exception when writing render data to file: ", e); + } } - else + else if (!LodUtil.isInterruptOrReject(ex)) { - if (ex instanceof InterruptedException) - { - // expected if the transformer is shut down, the exception can be ignored -// LOGGER.warn("RenderSource file transforming interrupted."); - - int ignoreEmptyWarning = 0; // explicitly handling these exceptions is important so we know where they are going and if there is an issue we can easily re-enable the logging - } - else if (ex instanceof RejectedExecutionException || ex.getCause() instanceof RejectedExecutionException) - { - // expected if the transformer was already shut down, the exception can be ignored -// LOGGER.warn("RenderSource file transforming interrupted."); - - int ignoreEmptyWarning = 0; - } - else if (!UncheckedInterruptedException.isInterrupt(ex)) - { - LOGGER.error("Exception when updating render file using data source: ", ex); - } + LOGGER.error("Exception when updating render file using data source: ", ex); } box.close(); transformationCompleteFuture.complete(null); }); return transformationCompleteFuture; } - - /** TODO at some point this method may need to be made "async" like {@link RenderSourceFileHandler#onReadRenderSourceLoadedFromCacheAsync} since the insides are async */ - public ColumnRenderSource onRenderFileLoaded(ColumnRenderSource renderSource, RenderMetaDataFile file) + + public CompletableFuture onRenderFileLoaded(ColumnRenderSource renderSource, RenderMetaDataFile file) { - this.updateCacheAsync(renderSource, file).join(); - return renderSource; + return this.updateCacheAsync(renderSource, file).handle((voidObj, ex) -> { + if (ex != null && !LodUtil.isInterruptOrReject(ex)) { + LOGGER.error("Exception when updating render file using data source: ", ex); + } + return renderSource; + }); } public CompletableFuture onReadRenderSourceLoadedFromCacheAsync(RenderMetaDataFile file, ColumnRenderSource data) { @@ -508,17 +499,18 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider // if the save futures didn't already complete, wait for them and then shut down the thread pool CompletableFuture combinedFuture = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); - combinedFuture.thenRun(() -> + combinedFuture.handle((result, ex) -> { + if (ex != null && !LodUtil.isInterruptOrReject(ex)) { + LOGGER.error("Exception when waiting for render source files to save", ex); + } + return null; + }).thenRun(() -> { LOGGER.info("Finished closing "+this.getClass().getSimpleName()+", ["+futures.size()+"] files were saved out of ["+this.filesBySectionPos.size()+"] total files."); this.fileHandlerThreadPool.shutdown(); }); } - - // if the save futures were already completed, the above "thenRun" won't fire, - // if the executor isn't currently running anything, shut it down - if (!this.fileHandlerThreadPool.isTerminated() && this.fileHandlerThreadPool.getActiveCount() == 0) - { + else { LOGGER.info("Finished closing " + this.getClass().getSimpleName() + " when files were already saved."); this.fileHandlerThreadPool.shutdown(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/BatchGenerator.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/BatchGenerator.java index 1fd4fc79f..b63d3a012 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/BatchGenerator.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/BatchGenerator.java @@ -138,7 +138,15 @@ public class BatchGenerator implements IDhApiWorldGenerator // the consumer needs to be wrapped like this because the API can't use DH core objects (and IChunkWrapper can't be easily put into the API project) Consumer consumerWrapper = (chunkWrapper) -> resultConsumer.accept(new Object[]{ chunkWrapper }); - return this.generationEnvironment.generateChunks(chunkPosMinX, chunkPosMinZ, genChunkSize, targetStep, worldGeneratorThreadPool, consumerWrapper); + try { + return this.generationEnvironment.generateChunks(chunkPosMinX, chunkPosMinZ, genChunkSize, targetStep, worldGeneratorThreadPool, consumerWrapper); + } + catch (Exception e) { + if (!LodUtil.isInterruptOrReject(e)) LOGGER.error("Error starting future for chunk generation", e); + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(e); + return future; + } } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 0e003dd77..2e1a1ad1d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -426,8 +426,7 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable if (exception != null) { // don't log the shutdown exceptions - if (!UncheckedInterruptedException.isInterrupt(exception) - && !(exception instanceof CancellationException || exception.getCause() instanceof CancellationException)) + if (!LodUtil.isInterruptOrReject(exception)) { LOGGER.error("Error generating data for section "+taskPos, exception); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java index 02bad3b97..1a96cef6a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.util; import java.util.Iterator; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletionException; import java.util.concurrent.RejectedExecutionException; @@ -308,7 +309,8 @@ public class LodUtil public static boolean isInterruptOrReject(Throwable t) { Throwable unwrapped = LodUtil.ensureUnwrap(t); return UncheckedInterruptedException.isInterrupt(unwrapped) || - unwrapped instanceof RejectedExecutionException; + unwrapped instanceof RejectedExecutionException || + unwrapped instanceof CancellationException; } } From 3190d607294b0136f2089c7c6410e0738844a813 Mon Sep 17 00:00:00 2001 From: coolGi Date: Mon, 17 Jul 2023 02:08:34 +0930 Subject: [PATCH 09/20] Slight fix to the reliability of the auto updater --- .../distanthorizons/core/config/Config.java | 2 +- .../core/jar/updater/SelfUpdater.java | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 05f70c537..2dbfebf26 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -878,7 +878,7 @@ public class Config .build(); public static ConfigEntry enableSilentUpdates = new ConfigEntry.Builder() - .set(true) + .set(false) .comment("" + "Should Distant Horizons silently, automatically download and install new versions? " + "") diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java index 54fa9a906..ec5b91592 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java @@ -10,7 +10,10 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.wrapperInterfaces.IVersionConstants; import org.apache.logging.log4j.Logger; +import java.io.File; +import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; import java.security.MessageDigest; /** @@ -23,6 +26,7 @@ public class SelfUpdater { /** As we cannot delete(or replace) the jar while the mod is running, we just have this to delete it once the game closes */ public static boolean deleteOldOnClose = false; + public static File newFileLocation; /** * Should be called on the game starting. @@ -49,8 +53,9 @@ public class SelfUpdater { LOGGER.info("New version ("+ModrinthGetter.getLatestNameForVersion(mcVersion)+") of "+ ModInfo.READABLE_NAME+" is available"); if (Config.Client.Advanced.AutoUpdater.enableSilentUpdates.get()) { + newFileLocation = JarUtils.jarFile.getParentFile().toPath().resolve("update").resolve(ModInfo.NAME + "-" + ModrinthGetter.getLatestNameForVersion(mcVersion) + ".jar").toFile(); // Auto-update mod - updateMod(mcVersion); + updateMod(mcVersion, newFileLocation); return false; } // else return true; @@ -62,6 +67,13 @@ public class SelfUpdater { */ public static void onClose() { if (deleteOldOnClose) { + try { + Files.move(newFileLocation.toPath(), JarUtils.jarFile.getParentFile().toPath().resolve(newFileLocation.getName())); + Files.delete(newFileLocation.getParentFile().toPath()); + } catch (Exception e) { + LOGGER.warn("Failed to move updated fire from ["+ newFileLocation.getAbsolutePath() +"] to ["+ JarUtils.jarFile.getParentFile().getAbsolutePath() +"], please move it manually"); + e.printStackTrace(); + } try { Files.delete(JarUtils.jarFile.toPath()); } catch (Exception e) { @@ -72,14 +84,18 @@ public class SelfUpdater { } public static boolean updateMod() { + String mcVer = SingletonInjector.INSTANCE.get(IVersionConstants.class).getMinecraftVersion(); + newFileLocation = JarUtils.jarFile.getParentFile().toPath().resolve("update").resolve(ModInfo.NAME + "-" + ModrinthGetter.getLatestNameForVersion(mcVer) + ".jar").toFile(); return updateMod( - SingletonInjector.INSTANCE.get(IVersionConstants.class).getMinecraftVersion() + mcVer, + newFileLocation ); } - public static boolean updateMod(String minecraftVersion) { + public static boolean updateMod(String minecraftVersion, File file) { try { LOGGER.info("Attempting to auto update " + ModInfo.READABLE_NAME); - WebDownloader.downloadAsFile(ModrinthGetter.getLatestDownloadForVersion(minecraftVersion), JarUtils.jarFile.getParentFile().toPath().resolve(ModInfo.NAME + "-" + ModrinthGetter.getLatestNameForVersion(minecraftVersion) + ".jar").toFile()); + Files.createDirectories(file.getParentFile().toPath()); + WebDownloader.downloadAsFile(ModrinthGetter.getLatestDownloadForVersion(minecraftVersion), file); deleteOldOnClose = true; LOGGER.info(ModInfo.READABLE_NAME + " successfully updated. It will apply on game's relaunch"); return true; From f6e68082b71d91c8b7ed04c27e3215ec09cb5ceb Mon Sep 17 00:00:00 2001 From: coolGi Date: Mon, 17 Jul 2023 02:14:17 +0930 Subject: [PATCH 10/20] Added checking for the checksum of the update --- .../distanthorizons/core/jar/updater/SelfUpdater.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java index ec5b91592..d08f3efa3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java @@ -94,13 +94,22 @@ public class SelfUpdater { public static boolean updateMod(String minecraftVersion, File file) { try { LOGGER.info("Attempting to auto update " + ModInfo.READABLE_NAME); + Files.createDirectories(file.getParentFile().toPath()); WebDownloader.downloadAsFile(ModrinthGetter.getLatestDownloadForVersion(minecraftVersion), file); + + // Check if the checksum of the downloaded jar is correct (not required, but good to have to prevent corruption or interception) + if (!JarUtils.getFileChecksum(MessageDigest.getInstance("SHA"), file).equals(ModrinthGetter.getLatestShaForVersion(minecraftVersion))) { + LOGGER.warn("DH update checksum failed, aborting install"); + throw new Exception("Checksum failed"); + } + deleteOldOnClose = true; + LOGGER.info(ModInfo.READABLE_NAME + " successfully updated. It will apply on game's relaunch"); return true; } catch (Exception e) { - LOGGER.info("Failed to update "+ModInfo.READABLE_NAME+" to version "+ModrinthGetter.getLatestNameForVersion(minecraftVersion)); + LOGGER.warn("Failed to update "+ModInfo.READABLE_NAME+" to version "+ModrinthGetter.getLatestNameForVersion(minecraftVersion)); e.printStackTrace(); return false; } From fe16c5a9cacf8200640b642f5830359b0b38bf57 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 13:38:00 -0500 Subject: [PATCH 11/20] Fix old thread pools not being shutdown when resized --- .../bufferBuilding/ColumnRenderBufferBuilder.java | 6 ++++++ .../transformers/DataRenderTransformer.java | 11 ++++++++++- .../core/file/fullDatafile/FullDataFileHandler.java | 11 ++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 9b788b029..27b636a19 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -387,6 +387,12 @@ public class ColumnRenderBufferBuilder } public static void setThreadPoolSize(int threadPoolSize) { + if (bufferBuilderThreadPool != null) + { + // close the previous thread pool if one exists + bufferBuilderThreadPool.shutdown(); + } + bufferBuilderThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Buffer Builder"); maxNumberOfConcurrentCalls = threadPoolSize * MAX_NUMBER_OF_CONCURRENT_CALLS_PER_THREAD; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java index 5b339d2a3..a51f7c9fc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java @@ -87,7 +87,16 @@ public class DataRenderTransformer setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads.get()); } } - public static void setThreadPoolSize(int threadPoolSize) { transformerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Full/Render Data Transformer"); } + public static void setThreadPoolSize(int threadPoolSize) + { + if (transformerThreadPool != null) + { + // close the previous thread pool if one exists + transformerThreadPool.shutdown(); + } + + transformerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Full/Render Data Transformer"); + } /** * Stops any executing tasks and destroys the executor.
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 66d5fcd4c..f81efb3ee 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -578,7 +578,16 @@ public class FullDataFileHandler implements IFullDataSourceProvider setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get()); } } - public static void setThreadPoolSize(int threadPoolSize) { fileHandlerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread"); } + public static void setThreadPoolSize(int threadPoolSize) + { + if (fileHandlerThreadPool != null) + { + // close the previous thread pool if one exists + fileHandlerThreadPool.shutdown(); + } + + fileHandlerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread"); + } /** * Stops any executing tasks and destroys the executor.
From c3ed5f98a5c13f0286fecdacc663379540d3141b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 17:05:45 -0500 Subject: [PATCH 12/20] Add and use Rate Limited thread pools --- .../distanthorizons/core/config/Config.java | 25 ++++++++ .../ColumnRenderBufferBuilder.java | 2 +- .../transformers/ChunkToLodBuilder.java | 2 +- .../transformers/DataRenderTransformer.java | 2 +- .../fullDatafile/FullDataFileHandler.java | 2 +- .../core/generation/WorldGenerationQueue.java | 11 +++- .../distanthorizons/core/util/ThreadUtil.java | 62 +++++++++--------- .../RateLimitedThreadPoolExecutor.java | 63 +++++++++++++++++++ .../assets/distanthorizons/lang/en_us.json | 14 +++++ 9 files changed, 150 insertions(+), 33 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 2dbfebf26..942063ace 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -737,6 +737,11 @@ public class Config + "CPU performance may suffer if Distant Horizons has a lot to load or generate. \n" + "This can be an issue when first loading into a world, when flying, and/or when generating new terrain."; + public static final String THREAD_RUN_TIME_RATIO_NOTE = "" + + "If this value is less than 1.0, it will be treated as a percentage \n" + + "of time a each thread can run before going idle. \n" + + ""; + public static final ConfigEntry numberOfWorldGenerationThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -753,6 +758,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForWorldGenerationThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static ConfigEntry numberOfBufferBuilderThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -767,6 +776,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForBufferBuilderThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static final ConfigEntry numberOfFileHandlerThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -781,6 +794,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForFileHandlerThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static final ConfigEntry numberOfDataConverterThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -798,6 +815,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForDataConverterThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); public static final ConfigEntry numberOfChunkLodConverterThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, @@ -811,6 +832,10 @@ public class Config + "\n" + THREAD_NOTE) .build(); + public static final ConfigEntry runTimeRatioForChunkLodConverterThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, 1.0, 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 27b636a19..d90cbec5e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -393,7 +393,7 @@ public class ColumnRenderBufferBuilder bufferBuilderThreadPool.shutdown(); } - bufferBuilderThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Buffer Builder"); + bufferBuilderThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "Buffer Builder", Config.Client.Advanced.MultiThreading.runTimeRatioForBufferBuilderThreads); maxNumberOfConcurrentCalls = threadPoolSize * MAX_NUMBER_OF_CONCURRENT_CALLS_PER_THREAD; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java index 5931b2ab1..5fde3959f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -230,7 +230,7 @@ public class ChunkToLodBuilder implements AutoCloseable } threadCount = threadPoolSize; - executorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, ChunkToLodBuilder.class); + executorThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, ChunkToLodBuilder.class.getSimpleName(), Config.Client.Advanced.MultiThreading.runTimeRatioForChunkLodConverterThreads); } /** diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java index a51f7c9fc..6f3802754 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java @@ -95,7 +95,7 @@ public class DataRenderTransformer transformerThreadPool.shutdown(); } - transformerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "Full/Render Data Transformer"); + transformerThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "Full/Render Data Transformer", Config.Client.Advanced.MultiThreading.runTimeRatioForDataConverterThreads); } /** diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index f81efb3ee..668a03cbf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -586,7 +586,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider fileHandlerThreadPool.shutdown(); } - fileHandlerThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread"); + fileHandlerThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, FullDataFileHandler.class.getSimpleName()+"Thread", Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads); } /** diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 2e1a1ad1d..228990d6c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -510,7 +510,16 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads.get()); } } - public static void setThreadPoolSize(int threadPoolSize) { worldGeneratorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, "DH-Gen-Worker-Thread", Thread.MIN_PRIORITY); } + public static void setThreadPoolSize(int threadPoolSize) + { + if (worldGeneratorThreadPool != null) + { + // close the previous thread pool if one exists + worldGeneratorThreadPool.shutdown(); + } + + worldGeneratorThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "DH-Gen-Worker-Thread", Thread.MIN_PRIORITY, Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads); + } /** * Stops any executing tasks and destroys the executor.
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java index 9e17a82d0..19fb369d2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java @@ -1,6 +1,9 @@ package com.seibel.distanthorizons.core.util; +import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; +import com.seibel.distanthorizons.core.config.types.ConfigEntry; import com.seibel.distanthorizons.core.util.objects.DhThreadFactory; +import com.seibel.distanthorizons.core.util.objects.RateLimitedThreadPoolExecutor; import java.util.concurrent.*; @@ -9,9 +12,33 @@ public class ThreadUtil public static int MINIMUM_RELATIVE_PRIORITY = -5; public static int DEFAULT_RELATIVE_PRIORITY = 0; + // TODO currently only used for RateLimitedThreadPools could this be used for all thread pools? + /** used to track and remove old listeners for certain pools if the thread pool is recreated. */ + private static final ConcurrentHashMap> THREAD_CHANGE_LISTENERS_BY_THREAD_NAME = new ConcurrentHashMap<>(); + // create rate limited thread pool // + + public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, ConfigEntry runTimeRatioConfigEntry) { return makeRateLimitedThreadPool(poolSize, name, 0, runTimeRatioConfigEntry); } + public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, int relativePriority, ConfigEntry runTimeRatioConfigEntry) + { + // remove the old listener if one exists + if (THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.containsKey(name)) + { + // note: this assumes only one thread pool exists with a given name + THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.get(name).close(); + THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.remove(name); + } + + RateLimitedThreadPoolExecutor executor = new RateLimitedThreadPoolExecutor(poolSize, runTimeRatioConfigEntry.get(), new DhThreadFactory("DH-" + name, Thread.NORM_PRIORITY+relativePriority)); + + ConfigChangeListener changeListener = new ConfigChangeListener<>(runTimeRatioConfigEntry, (newRunTimeRatio) -> { executor.runTimeRatio = newRunTimeRatio; }); + THREAD_CHANGE_LISTENERS_BY_THREAD_NAME.put(name, changeListener); + + return executor; + } + // create thread pool // @@ -26,38 +53,17 @@ public class ThreadUtil new DhThreadFactory("DH-" + name, Thread.NORM_PRIORITY+relativePriority)); } - public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz, int relativePriority) - { - return makeThreadPool(poolSize, clazz.getSimpleName(), relativePriority); - } - public static ThreadPoolExecutor makeThreadPool(int poolSize, String name) - { - return makeThreadPool(poolSize, name, 0); - } - public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz) - { - return makeThreadPool(poolSize, clazz.getSimpleName(), 0); - } + public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz, int relativePriority) { return makeThreadPool(poolSize, clazz.getSimpleName(), relativePriority); } + public static ThreadPoolExecutor makeThreadPool(int poolSize, String name) { return makeThreadPool(poolSize, name, 0); } + public static ThreadPoolExecutor makeThreadPool(int poolSize, Class clazz) { return makeThreadPool(poolSize, clazz.getSimpleName(), 0); } // create single thread pool // - public static ThreadPoolExecutor makeSingleThreadPool(String name, int relativePriority) - { - return makeThreadPool(1, name, relativePriority); - } - public static ThreadPoolExecutor makeSingleThreadPool(Class clazz, int relativePriority) - { - return makeThreadPool(1, clazz.getSimpleName(), relativePriority); - } - public static ThreadPoolExecutor makeSingleThreadPool(String name) - { - return makeThreadPool(1, name, 0); - } - public static ThreadPoolExecutor makeSingleThreadPool(Class clazz) - { - return makeThreadPool(1, clazz.getSimpleName(), 0); - } + public static ThreadPoolExecutor makeSingleThreadPool(String name, int relativePriority) { return makeThreadPool(1, name, relativePriority); } + public static ThreadPoolExecutor makeSingleThreadPool(Class clazz, int relativePriority) { return makeThreadPool(1, clazz.getSimpleName(), relativePriority); } + public static ThreadPoolExecutor makeSingleThreadPool(String name) { return makeThreadPool(1, name, 0); } + public static ThreadPoolExecutor makeSingleThreadPool(Class clazz) { return makeThreadPool(1, clazz.getSimpleName(), 0); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java new file mode 100644 index 000000000..7b23a6dc4 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/RateLimitedThreadPoolExecutor.java @@ -0,0 +1,63 @@ +package com.seibel.distanthorizons.core.util.objects; + +import java.util.concurrent.*; + +/** + * Can be used to more finely control CPU usage and + * reduce CPU usage if only 1 thread is already assigned. + */ +public class RateLimitedThreadPoolExecutor extends ThreadPoolExecutor +{ + public volatile double runTimeRatio; + + /** When this thread started running its last task */ + private final ThreadLocal runStartNanoTimeRef = ThreadLocal.withInitial(() -> -1L); + /** How long it took this thread to run its last task */ + private final ThreadLocal lastRunDurationNanoTimeRef = ThreadLocal.withInitial(() -> -1L); + + + + //==============// + // constructors // + //==============// + + public RateLimitedThreadPoolExecutor(int corePoolSize, double runTimeRatio, ThreadFactory threadFactory) + { + super(corePoolSize, corePoolSize, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + threadFactory); + + this.runTimeRatio = runTimeRatio; + } + + + + //===========// + // overrides // + //===========// + + protected void beforeExecute(Thread thread, Runnable runnable) + { + super.beforeExecute(thread, runnable); + + if (this.runTimeRatio < 1.0 && this.lastRunDurationNanoTimeRef.get() != -1) + { + try + { + long deltaMs = TimeUnit.NANOSECONDS.toMillis(this.lastRunDurationNanoTimeRef.get()); + Thread.sleep((long) (deltaMs/this.runTimeRatio - deltaMs)); + } catch (InterruptedException ignored) { } + } + + this.runStartNanoTimeRef.set(System.nanoTime()); + } + + @Override + protected void afterExecute(Runnable runnable, Throwable throwable) + { + super.afterExecute(runnable, throwable); + this.lastRunDurationNanoTimeRef.set(System.nanoTime() - this.runStartNanoTimeRef.get()); + } + +} \ No newline at end of file diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 94d2bbc46..3ee414af4 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -306,22 +306,36 @@ "NO. of world generation threads", "distanthorizons.config.client.advanced.multiThreading.numberOfWorldGenerationThreads.@tooltip": "How many threads should be used when generating LODs \n outside the normal render distance? \n\nIf it this is less than 1, it will be treated as a percentage \nof time a single thread can run before going idle. \n\nIf you experience stuttering when generating distant LODs, \ndecrease this number. If you want to increase LOD \ngeneration speed, increase this number. \n\nThis and the number of buffer builder threads are independent, \nif they add up to more threads than your CPU has cores \nthat is ok.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForWorldGenerationThreads": + "Runtime % for world generation threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfBufferBuilderThreads": "NO. of buffer builder threads", "distanthorizons.config.client.advanced.multiThreading.numberOfBufferBuilderThreads.@tooltip": "The number of threads used when building geometry data. \nCan only be between 1 and your CPU's processor count.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForBufferBuilderThreads": + "Runtime % for buffer builder threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfFileHandlerThreads": "NO. of file handler threads", "distanthorizons.config.client.advanced.multiThreading.numberOfFileHandlerThreads.@tooltip": "The number of threads used when building vertex buffers \n(The things sent to your GPU to draw the LODs). \nCan only be between 1 and your CPU's processor count.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForFileHandlerThreads": + "Runtime % for file handler threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfDataConverterThreads": "NO. of data converter threads", "distanthorizons.config.client.advanced.multiThreading.numberOfDataConverterThreads.@tooltip": "The number of threads used when converting ID data to render-able data. \n(This generally happens when generating new terrain or changing graphics settings). \nCan only be between 1 and your CPU's processor count.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForDataConverterThreads": + "Runtime % for data converter threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads": "NO. of chunk LOD converter threads", "distanthorizons.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads.@tooltip": "How many threads should be used to convert Minecraft chunks into LOD data? \nThese threads run both when terrain is generated and when \nchunks are loaded, unloaded, and modified.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForChunkLodConverterThreads": + "Runtime % for chunk LOD converter threads", From a7a54598b29dcc035f6dd214c62df43370d967b9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 17:41:32 -0500 Subject: [PATCH 13/20] Add Thread runTime to the thread presets --- .../distanthorizons/core/config/Config.java | 19 ++-- .../ThreadPresetConfigEventHandler.java | 90 ++++++++++++++++--- 2 files changed, 88 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 942063ace..c9c98cb98 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -739,8 +739,11 @@ public class Config public static final String THREAD_RUN_TIME_RATIO_NOTE = "" + "If this value is less than 1.0, it will be treated as a percentage \n" - + "of time a each thread can run before going idle. \n" - + ""; + + "of time each thread can run before going idle. \n" + + "\n" + + "This can be used to reduce CPU usage if the thread count \n" + + "is already set to 1 for the given option, or more finely \n" + + "tune CPU performance."; public static final ConfigEntry numberOfWorldGenerationThreads = new ConfigEntry.Builder() @@ -759,7 +762,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForWorldGenerationThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getWorldGenDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); @@ -777,7 +780,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForBufferBuilderThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getBufferBuilderDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); @@ -795,7 +798,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForFileHandlerThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getFileHandlerDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); @@ -816,13 +819,13 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForDataConverterThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getDataConverterDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); public static final ConfigEntry numberOfChunkLodConverterThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, - ThreadPresetConfigEventHandler.getChunkLodConvertersDefaultThreadCount(), + ThreadPresetConfigEventHandler.getChunkLodConverterDefaultThreadCount(), Runtime.getRuntime().availableProcessors()) .comment("" + "How many threads should be used to convert Minecraft chunks into LOD data? \n" @@ -833,7 +836,7 @@ public class Config + THREAD_NOTE) .build(); public static final ConfigEntry runTimeRatioForChunkLodConverterThreads = new ConfigEntry.Builder() - .setMinDefaultMax(0.01, 1.0, 1.0) + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getChunkLodConverterDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index 2c60d3ea9..5bdbc3722 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -20,8 +20,9 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan private static final Logger LOGGER = LogManager.getLogger(); + public static int getWorldGenDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions worldGen = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, + private final ConfigEntryWithPresetOptions worldGenThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -30,9 +31,20 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getWorldGenDefaultRunTimeRatio() { return 0.25; } + private final ConfigEntryWithPresetOptions worldGenRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.1); + this.put(EThreadPreset.LOW_IMPACT, getWorldGenDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.5); + this.put(EThreadPreset.AGGRESSIVE, 0.75); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); + public static int getBufferBuilderDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions bufferBuilders = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfBufferBuilderThreads, + private final ConfigEntryWithPresetOptions bufferBuilderThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfBufferBuilderThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -41,9 +53,20 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getBufferBuilderDefaultRunTimeRatio() { return 0.5; } + private final ConfigEntryWithPresetOptions bufferBuilderRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForBufferBuilderThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EThreadPreset.LOW_IMPACT, getBufferBuilderDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); + public static int getFileHandlerDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions fileHandlers = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, + private final ConfigEntryWithPresetOptions fileHandlerThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -52,9 +75,20 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getFileHandlerDefaultRunTimeRatio() { return 0.5; } + private final ConfigEntryWithPresetOptions fileHandlerRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EThreadPreset.LOW_IMPACT, getFileHandlerDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); + public static int getDataConverterDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions dataConverters = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads, + private final ConfigEntryWithPresetOptions dataConverterThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -63,17 +97,38 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getDataConverterDefaultRunTimeRatio() { return 0.25; } + private final ConfigEntryWithPresetOptions dataConverterRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForDataConverterThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.1); + this.put(EThreadPreset.LOW_IMPACT, getDataConverterDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); - public static int getChunkLodConvertersDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions chunkLodConverters = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads, + + public static int getChunkLodConverterDefaultThreadCount() { return getThreadCountByPercent(0.1); } + private final ConfigEntryWithPresetOptions chunkLodConverterThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); - this.put(EThreadPreset.LOW_IMPACT, getChunkLodConvertersDefaultThreadCount()); + this.put(EThreadPreset.LOW_IMPACT, getChunkLodConverterDefaultThreadCount()); this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.2)); this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); + public static double getChunkLodConverterDefaultRunTimeRatio() { return 0.5; } + private final ConfigEntryWithPresetOptions chunkLodConverterRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForChunkLodConverterThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EThreadPreset.LOW_IMPACT, getDataConverterDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.75); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); @@ -85,16 +140,25 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan private ThreadPresetConfigEventHandler() { // add each config used by this preset - this.configList.add(this.worldGen); - this.configList.add(this.bufferBuilders); - this.configList.add(this.fileHandlers); - this.configList.add(this.dataConverters); - this.configList.add(this.chunkLodConverters); + this.configList.add(this.worldGenThreadCount); + this.configList.add(this.worldGenRunTime); + + this.configList.add(this.bufferBuilderThreadCount); + this.configList.add(this.bufferBuilderRunTime); + + this.configList.add(this.fileHandlerThreadCount); + this.configList.add(this.fileHandlerRunTime); + + this.configList.add(this.dataConverterThreadCount); + this.configList.add(this.dataConverterRunTime); + + this.configList.add(this.chunkLodConverterThreadCount); + this.configList.add(this.chunkLodConverterRunTime); for (ConfigEntryWithPresetOptions config : this.configList) { - // ignore try-using, the listener should only ever be added once and should never be removed + // ignore try-using, the listeners should only ever be added once and should never be removed new ConfigChangeListener<>(config.configEntry, (val) -> { this.onConfigValueChanged(); }); } } From a00e1c22c3058053793fae7e701275519fec7c89 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 20:56:34 -0500 Subject: [PATCH 14/20] Full Data source refactoring --- .../fullDatafile/FullDataFileHandler.java | 53 ++++++++----------- .../file/fullDatafile/FullDataMetaFile.java | 34 +++++++----- .../GeneratedFullDataFileHandler.java | 3 +- 3 files changed, 43 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 668a03cbf..81ec52f95 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -30,7 +30,6 @@ import java.util.function.Consumer; import java.util.function.Function; import static com.seibel.distanthorizons.core.util.FileScanUtil.LOD_FILE_POSTFIX; -import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX; public class FullDataFileHandler implements IFullDataSourceProvider { @@ -40,11 +39,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected static ExecutorService fileHandlerThreadPool; protected static ConfigChangeListener configListener; - + private final ConcurrentHashMap unloadedFiles = new ConcurrentHashMap<>(); private final ConcurrentHashMap fileBySectionPos = new ConcurrentHashMap<>(); public void ForEachFile(Consumer consumer) { this.fileBySectionPos.values().forEach(consumer); } - + + private LinkedList> onUpdatedListeners = new LinkedList<>(); + protected final IDhLevel level; protected final File saveDir; /** @@ -59,6 +60,11 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected final AtomicInteger topDetailLevel = new AtomicInteger(DhSectionPos.SECTION_REGION_DETAIL_LEVEL); protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; + + //=============// + // constructor // + //=============// + public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this.level = level; @@ -69,7 +75,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider } FileScanUtil.scanFiles(saveStructure, level.getLevelWrapper(), this, null); } - + + // constructor helpers // + /** * Caller must ensure that this method is called only once, * and that the {@link FullDataFileHandler} is not used before this method is called. @@ -179,6 +187,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider } } + + + protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile) { FullDataMetaFile metaFile = this.fileBySectionPos.get(pos); @@ -414,41 +425,19 @@ public class FullDataFileHandler implements IFullDataSourceProvider } return metaFile.flushAndSaveAsync(); } - - - private LinkedList> onUpdatedListeners = new LinkedList<>(); + + @Override public synchronized void addOnUpdatedListener(Consumer listener) { this.onUpdatedListeners.add(listener); } - -// @Override -// public long getCacheVersion(DhSectionPos sectionPos) -// { -// FullDataMetaFile file = this.files.get(sectionPos); -// if (file == null) -// { -// return 0; -// } -// return file.getCacheVersion(); -// } -// @Override -// public boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion) -// { -// FullDataMetaFile file = this.files.get(sectionPos); -// if (file == null) -// { -// return cacheVersion >= 0; -// } -// return file.isCacheVersionValid(cacheVersion); -// } - - protected IIncompleteFullDataSource makeDataSource(DhSectionPos pos) + protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { return pos.sectionDetailLevel <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL ? - HighDetailIncompleteFullDataSource.createEmpty(pos) : LowDetailIncompleteFullDataSource.createEmpty(pos); + HighDetailIncompleteFullDataSource.createEmpty(pos) : + LowDetailIncompleteFullDataSource.createEmpty(pos); } protected CompletableFuture sampleFromFiles(IIncompleteFullDataSource source, ArrayList existingFiles) @@ -486,7 +475,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider public CompletableFuture onCreateDataFile(FullDataMetaFile file) { DhSectionPos pos = file.pos; - IIncompleteFullDataSource source = this.makeDataSource(pos); + IIncompleteFullDataSource source = this.makeEmptyDataSource(pos); ArrayList existFiles = new ArrayList<>(); ArrayList missing = new ArrayList<>(); this.getDataFilesForPosition(pos, pos, existFiles, missing); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java index 8cd060da7..9ec036858 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java @@ -254,9 +254,9 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I }, executorService)); } - private void makeCreateCompletionStage(ExecutorService executorService, CompletableFuture completer) + private void makeCreateCompletionStage(CompletableFuture completer) { - makeUpdateCompletionStage(completer, this.fullDataSourceProvider.onCreateDataFile(this) + this.makeUpdateCompletionStage(completer, this.fullDataSourceProvider.onCreateDataFile(this) .thenApply((fullDataSource) -> { this.baseMetaData = this._makeBaseMetaData(fullDataSource); @@ -273,31 +273,39 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I CacheQueryResult result = this.getCachedDataSourceAsync(); - if (result.needsLoad) { - LodUtil.assertTrue(!inCrit); - inCrit = true; + if (result.needsLoad) + { + LodUtil.assertTrue(!this.inCrit); + this.inCrit = true; CompletableFuture future = result.future; // don't continue if the provider has been shut down ExecutorService executorService = this.fullDataSourceProvider.getIOExecutor(); if (executorService.isTerminated()) { - inCrit = false; - dataSourceLoadFutureRef.set(null); + this.inCrit = false; + this.dataSourceLoadFutureRef.set(null); future.complete(null); return future; } // create a new Meta file - if (!doesFileExist) { - makeCreateCompletionStage(executorService, future); + if (!this.doesFileExist) + { + this.makeCreateCompletionStage(future); } - // Otherwise, load and update file - else { - if (this.baseMetaData == null) throw new IllegalStateException("Meta data not loaded!"); - makeLoadCompletionStage(executorService, future); + else + { + // Otherwise, load and update file + if (this.baseMetaData == null) + { + throw new IllegalStateException("Meta data not loaded!"); + } + + this.makeLoadCompletionStage(executorService, future); } } + return result.future; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 6f9631bbf..dd315783b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -17,7 +17,6 @@ import com.seibel.distanthorizons.core.util.LodUtil; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; -import java.io.File; import java.lang.ref.WeakReference; import java.util.*; import java.util.concurrent.*; @@ -180,8 +179,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public CompletableFuture onCreateDataFile(FullDataMetaFile file) { DhSectionPos pos = file.pos; - IIncompleteFullDataSource data = makeDataSource(pos); CompletableFuture future = updateDataGenStatus(file, data); + IIncompleteFullDataSource data = makeEmptyDataSource(pos); // Cant start gen task, so return the data return future == null ? CompletableFuture.completedFuture(data) : future; } From f0d3ece345556aee1021be5ab23c054df482ff64 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 21:52:20 -0500 Subject: [PATCH 15/20] Full Data source refactoring 2 --- .../fullDatafile/FullDataFileHandler.java | 35 ++++++++------ .../GeneratedFullDataFileHandler.java | 46 +++++++++++-------- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 81ec52f95..010314c36 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -248,8 +248,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider return null; } - // This is a CAS with expected null value. this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + + // This is a CAS with expected null value. FullDataMetaFile metaFileCas = this.fileBySectionPos.putIfAbsent(pos, metaFile); return metaFileCas == null ? metaFile : metaFileCas; } @@ -266,6 +267,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider byte sectionDetail = posAreaToGet.sectionDetailLevel; boolean allEmpty = true; + // get all existing files for this position outerLoop: while (--sectionDetail >= this.minDetailLevel) { @@ -439,28 +441,33 @@ public class FullDataFileHandler implements IFullDataSourceProvider HighDetailIncompleteFullDataSource.createEmpty(pos) : LowDetailIncompleteFullDataSource.createEmpty(pos); } - - protected CompletableFuture sampleFromFiles(IIncompleteFullDataSource source, ArrayList existingFiles) + + /** populates the given data source using the given array of files */ + protected CompletableFuture sampleFromFileArray(IIncompleteFullDataSource recipientFullDataSource, ArrayList existingFiles) { // read in the existing data final ArrayList> loadDataFutures = new ArrayList<>(existingFiles.size()); for (FullDataMetaFile existingFile : existingFiles) { loadDataFutures.add(existingFile.loadOrGetCachedDataSourceAsync() - .exceptionally((ex) -> /*Ignore file read errors*/null) - .thenAccept((fullDataSource) -> + .exceptionally((ex) -> /*Ignore file read errors*/null) + .thenAccept((existingFullDataSource) -> + { + if (existingFullDataSource == null) { - if (fullDataSource == null) return; - //this.checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource); - //LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); - source.sampleFrom(fullDataSource); - }) + return; + } + + //LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); + recipientFullDataSource.sampleFrom(existingFullDataSource); + }) ); } - return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0])).thenApply(v -> source); + return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0])).thenApply(voidObj -> recipientFullDataSource); } - protected void makeFiles(ArrayList posList, ArrayList output) { + protected void makeFiles(ArrayList posList, ArrayList output) + { for (DhSectionPos missingPos : posList) { FullDataMetaFile newFile = this.getLoadOrMakeFile(missingPos, true); @@ -487,8 +494,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider } else { - makeFiles(missing, existFiles); - return sampleFromFiles(source, existFiles).thenApply(IIncompleteFullDataSource::tryPromotingToCompleteDataSource) + this.makeFiles(missing, existFiles); + return this.sampleFromFileArray(source, existFiles).thenApply(IIncompleteFullDataSource::tryPromotingToCompleteDataSource) .exceptionally((e) -> { FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, file, e); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index dd315783b..5e9c3d96e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -120,7 +120,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler ArrayList existingFiles = new ArrayList<>(); byte sectDetailLevel = (byte) (DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + maxSectDataDetailLevel); pos.forEachChildAtLevel(sectDetailLevel, p -> existingFiles.add(getLoadOrMakeFile(p, true))); - return sampleFromFiles(dataSource, existingFiles).thenApply(this::tryPromoteDataSource) + return sampleFromFileArray(dataSource, existingFiles).thenApply(this::tryPromoteDataSource) .exceptionally((e) -> { FullDataMetaFile newMetaFile = removeCorruptedFile(pos, file, e); @@ -152,26 +152,28 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // Try update the gen queue on this data source. If null, then nothing was done. @Nullable - private CompletableFuture updateDataGenStatus(FullDataMetaFile file, IIncompleteFullDataSource data) + private CompletableFuture updateFromExistingDataSources(FullDataMetaFile file, IIncompleteFullDataSource data) { DhSectionPos pos = file.pos; ArrayList existingFiles = new ArrayList<>(); ArrayList missingPositions = new ArrayList<>(); this.getDataFilesForPosition(pos, pos, existingFiles, missingPositions); - - if (missingPositions.size() == 1) { + + if (missingPositions.size() == 1) + { // Only missing myself. I.e. no child file data exists yet. - return tryStartGenTask(file, data); + return this.tryStartGenTask(file, data); } - else { - // Has stuff to sample. - makeFiles(missingPositions, existingFiles); - return sampleFromFiles(data, existingFiles).thenApply(this::tryPromoteDataSource) - .exceptionally((e) -> - { - FullDataMetaFile newMetaFile = removeCorruptedFile(pos, file, e); - return null; - }); + else + { + // There are other data source files to sample from. + this.makeFiles(missingPositions, existingFiles); + return this.sampleFromFileArray(data, existingFiles).thenApply(this::tryPromoteDataSource) + .exceptionally((e) -> + { + this.removeCorruptedFile(pos, file, e); + return null; + }); } } @@ -179,8 +181,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public CompletableFuture onCreateDataFile(FullDataMetaFile file) { DhSectionPos pos = file.pos; - CompletableFuture future = updateDataGenStatus(file, data); IIncompleteFullDataSource data = makeEmptyDataSource(pos); + CompletableFuture future = updateFromExistingDataSources(file, data); // Cant start gen task, so return the data return future == null ? CompletableFuture.completedFuture(data) : future; } @@ -205,12 +207,16 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } this.fireOnGenPosSuccessListeners(source.getSectionPos()); - if (source instanceof IIncompleteFullDataSource && !file.genQueueChecked) { + if (source instanceof IIncompleteFullDataSource && !file.genQueueChecked) + { WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null) { - CompletableFuture future = updateDataGenStatus(file, (IIncompleteFullDataSource) source); - if (future != null) { - return future.thenApply((newSource) -> { + if (worldGenQueue != null) + { + CompletableFuture future = this.updateFromExistingDataSources(file, (IIncompleteFullDataSource) source); + if (future != null) + { + return future.thenApply((newSource) -> + { onUpdated.accept(newSource); return newSource; }); From 6acb1a2e9f28a3de5064cc1e410c685bf6a1f515 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 21:56:05 -0500 Subject: [PATCH 16/20] Fix missing low detail LODs if the file didn't exist --- .../file/fullDatafile/FullDataFileHandler.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 010314c36..286e1ead4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -288,7 +288,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider continue; } - if (this.fileBySectionPos.containsKey(subPos)) + // check if a file for this pos exists, either loaded and unloaded + if (this.fileBySectionPos.containsKey(subPos) || this.unloadedFiles.containsKey(subPos)) { allEmpty = false; break outerLoop; @@ -317,6 +318,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider DhSectionPos childPos = pos.getChildByIndex(childIndex); if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos)) { + // load the file if it isn't already + if (this.unloadedFiles.containsKey(childPos)) + { + this.getLoadOrMakeFile(childPos, true); + } + + FullDataMetaFile metaFile = this.fileBySectionPos.get(childPos); if (metaFile != null) { @@ -386,12 +394,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData) { FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos); -// if (metaFile == null && sectionPos.sectionDetailLevel <= this.topDetailLevel.get()) -// { -// // create a new file if one doesn't exist, -// // this is done so we don't end up with holes where LODs should have been generated -// metaFile = this.getLoadOrMakeFile(sectionPos, true); -// } if (metaFile != null) { // there is a file for this position From 9d518a5fd662392feeb4379917c21bdd97d638bc Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 16 Jul 2023 22:07:59 -0500 Subject: [PATCH 17/20] fully remove band-aid fix for missing low detail LODs --- .../core/file/fullDatafile/FullDataFileHandler.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 286e1ead4..b1ddbda49 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -48,16 +48,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected final IDhLevel level; protected final File saveDir; - /** - * The starting value here denotes how far into the tree LOD writes should occur.
- * This is a band-aid fix to prevent lower detail sections from not being generated until the detail level - * is requested.

- * - * Note: that problem may still happen at a sufficiently large render distance, - * however this should kick the problem down the road - * far enough that it can be ignored for now. - */ - protected final AtomicInteger topDetailLevel = new AtomicInteger(DhSectionPos.SECTION_REGION_DETAIL_LEVEL); + protected final AtomicInteger topDetailLevel = new AtomicInteger(0); protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; From 388ea9050faf2279e1c6496f4d2b69499f16057e Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Mon, 17 Jul 2023 18:44:42 +0800 Subject: [PATCH 18/20] Fixed the mem leak issue by nuking the whole save-when-exit --- .../file/renderfile/RenderMetaDataFile.java | 6 +- .../renderfile/RenderSourceFileHandler.java | 106 ++++++++++++++---- 2 files changed, 90 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java index 645c55678..16b15d04d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java @@ -146,7 +146,11 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements { return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save. } - return source.thenAccept((columnRenderSource) -> { }); // Otherwise, wait for the data to be read (which also flushes changes to the file). + return source.handle((columnRenderSource, ex) -> { + if (ex != null && !LodUtil.isInterruptOrReject(ex)) + LOGGER.error("Failed to load render source for "+this.pos+" for flush and saving", ex); + return null; + }); // Otherwise, wait for the data to be read (which also flushes changes to the file). } private CacheQueryResult getOrStartCachedDataSourceAsync() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index adfcf7c55..3e238ba8e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -4,7 +4,9 @@ import com.google.common.collect.HashMultimap; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; +import com.seibel.distanthorizons.core.level.ClientLevelModule; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; @@ -38,6 +40,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private final ThreadPoolExecutor fileHandlerThreadPool; + private final F3Screen.NestedMessage threadPoolMsg; private final ConcurrentHashMap unloadedFiles = new ConcurrentHashMap<>(); private final ConcurrentHashMap filesBySectionPos = new ConcurrentHashMap<>(); @@ -47,6 +50,12 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider /** This is the lowest (highest numeric) detail level that this {@link RenderSourceFileHandler} is keeping track of. */ AtomicInteger topDetailLevel = new AtomicInteger(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); private final IFullDataSourceProvider fullDataSourceProvider; + + enum TaskType { + Read, UpdateReadData, Update, OnLoaded, + } + + private final WeakHashMap, TaskType> taskTracker = new WeakHashMap<>(); @@ -63,11 +72,48 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider { LOGGER.warn("Unable to create render data folder, file saving may fail."); } - this.fileHandlerThreadPool = ThreadUtil.makeSingleThreadPool("Render Source File Handler ["+this.level.getClientLevelWrapper().getDimensionType().getDimensionName()+"]"); + fileHandlerThreadPool = ThreadUtil.makeSingleThreadPool("Render Source File Handler ["+this.level.getClientLevelWrapper().getDimensionType().getDimensionName()+"]"); + + + this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log); FileScanUtil.scanFiles(saveStructure, level.getLevelWrapper(), null, this); } - + + /** Returns what should be displayed in Minecraft's F3 debug menu */ + private String[] f3Log() + { + ArrayList lines = new ArrayList<>(); + lines.add("Render Source File Handler ["+this.level.getClientLevelWrapper().getDimensionType().getDimensionName()+"]"); + lines.add(" Loaded files: "+this.filesBySectionPos.size() + " / " + (this.unloadedFiles.size() + this.filesBySectionPos.size())); + lines.add(" Thread pool tasks: "+fileHandlerThreadPool.getQueue().size() + " (completed: " + fileHandlerThreadPool.getCompletedTaskCount() + ")"); + + int totalFutures = taskTracker.size(); + EnumMap tasksOutstanding = new EnumMap<>(TaskType.class); + EnumMap tasksCompleted = new EnumMap<>(TaskType.class); + for (TaskType type : TaskType.values()) + { + tasksOutstanding.put(type, 0); + tasksCompleted.put(type, 0); + } + + synchronized (taskTracker) { + for (Map.Entry, TaskType> entry : taskTracker.entrySet()) { + if (entry.getKey().isDone()) { + tasksCompleted.put(entry.getValue(), tasksCompleted.get(entry.getValue()) + 1); + } else { + tasksOutstanding.put(entry.getValue(), tasksOutstanding.get(entry.getValue()) + 1); + } + } + } + int totalOutstanding = tasksOutstanding.values().stream().mapToInt(Integer::intValue).sum(); + lines.add(" Futures: "+totalFutures + " (outstanding: " + totalOutstanding + ")"); + for (TaskType type : TaskType.values()) + { + lines.add(" " + type + ": " + tasksOutstanding.get(type) + " / " + (tasksOutstanding.get(type) + tasksCompleted.get(type))); + } + return lines.toArray(new String[0]); + } //===============// @@ -291,8 +337,8 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider // On error, (when it returns null,) return an empty render source if (metaFile == null) return CompletableFuture.completedFuture(ColumnRenderSource.createEmptyRenderSource(pos)); - - return metaFile.loadOrGetCachedDataSourceAsync(this.fileHandlerThreadPool, this.level).handle( + + CompletableFuture future = metaFile.loadOrGetCachedDataSourceAsync(this.fileHandlerThreadPool, this.level).handle( (renderSource, exception) -> { if (exception != null) @@ -302,6 +348,10 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider return (renderSource != null) ? renderSource : ColumnRenderSource.createEmptyRenderSource(pos); }); + synchronized (taskTracker) { + taskTracker.put(future, TaskType.Read); + } + return future; } public CompletableFuture onCreateRenderFileAsync(RenderMetaDataFile file) @@ -391,13 +441,14 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider return null; }); - // future returned - CompletableFuture transformationCompleteFuture = new CompletableFuture<>(); - + synchronized (taskTracker) { + taskTracker.put(fullDataSourceFuture, TaskType.UpdateReadData); + } + // convert the full data source into a render source //LOGGER.info("Recreating cache for {}", data.getSectionPos()); - DataRenderTransformer.transformDataSourceAsync(fullDataSourceFuture, this.level) - .whenComplete((newRenderSource, ex) -> + CompletableFuture transformFuture = DataRenderTransformer.transformDataSourceAsync(fullDataSourceFuture, this.level) + .handle((newRenderSource, ex) -> { if (ex == null) { @@ -411,20 +462,30 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider { LOGGER.error("Exception when updating render file using data source: ", ex); } + else { + //LOGGER.info("Interrupted update of render file using data source: ", ex); + } box.close(); - transformationCompleteFuture.complete(null); + return null; }); - return transformationCompleteFuture; + synchronized (taskTracker) { + taskTracker.put(transformFuture, TaskType.Update); + } + return transformFuture; } public CompletableFuture onRenderFileLoaded(ColumnRenderSource renderSource, RenderMetaDataFile file) { - return this.updateCacheAsync(renderSource, file).handle((voidObj, ex) -> { + CompletableFuture future = this.updateCacheAsync(renderSource, file).handle((voidObj, ex) -> { if (ex != null && !LodUtil.isInterruptOrReject(ex)) { LOGGER.error("Exception when updating render file using data source: ", ex); } return renderSource; }); + synchronized (taskTracker) { + taskTracker.put(future, TaskType.OnLoaded); + } + return future; } public CompletableFuture onReadRenderSourceLoadedFromCacheAsync(RenderMetaDataFile file, ColumnRenderSource data) { @@ -475,17 +536,19 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider //=====================// // clearing / shutdown // //=====================// - + + //private static CompletableFuture cleanupTask; + @Override public void close() { LOGGER.info("Closing "+this.getClass().getSimpleName()+" with ["+this.filesBySectionPos.size()+"] files..."); - + /* // queue the file save futures ArrayList> futures = new ArrayList<>(); for (RenderMetaDataFile metaFile : this.filesBySectionPos.values()) { - CompletableFuture saveFuture = metaFile.flushAndSaveAsync(this.fileHandlerThreadPool); + CompletableFuture saveFuture = metaFile.flushAndSaveAsync(fileHandlerThreadPool); if (!saveFuture.isDone()) { futures.add(saveFuture); @@ -499,7 +562,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider // if the save futures didn't already complete, wait for them and then shut down the thread pool CompletableFuture combinedFuture = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); - combinedFuture.handle((result, ex) -> { + cleanupTask = combinedFuture.handle((result, ex) -> { if (ex != null && !LodUtil.isInterruptOrReject(ex)) { LOGGER.error("Exception when waiting for render source files to save", ex); } @@ -507,13 +570,14 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider }).thenRun(() -> { LOGGER.info("Finished closing "+this.getClass().getSimpleName()+", ["+futures.size()+"] files were saved out of ["+this.filesBySectionPos.size()+"] total files."); - this.fileHandlerThreadPool.shutdown(); + fileHandlerThreadPool.shutdown(); + threadPoolMsg.close(); }); } - else { - LOGGER.info("Finished closing " + this.getClass().getSimpleName() + " when files were already saved."); - this.fileHandlerThreadPool.shutdown(); - } + else {*/ + fileHandlerThreadPool.shutdown(); + threadPoolMsg.close(); + //} } From 5db5bce308189ddacc610e98c5e34a27e9311052 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 17 Jul 2023 21:43:38 -0500 Subject: [PATCH 19/20] rename runTimeRatioForDataConverterThreads -> ...TransformerThreads --- .../config/client/DhApiMultiThreadingConfig.java | 2 +- .../com/seibel/distanthorizons/core/config/Config.java | 4 ++-- .../presets/ThreadPresetConfigEventHandler.java | 8 ++++---- .../transformers/DataRenderTransformer.java | 7 ++++--- .../resources/assets/distanthorizons/lang/en_us.json | 10 +++++----- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiThreadingConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiThreadingConfig.java index b9e0e593d..0f38b0f59 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiThreadingConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiThreadingConfig.java @@ -46,7 +46,7 @@ public class DhApiMultiThreadingConfig implements IDhApiMultiThreadingConfig @Override public IDhApiConfigValue dataConverterThreads() - { return new DhApiConfigValue(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads); } + { return new DhApiConfigValue(Config.Client.Advanced.MultiThreading.numberOfDataTransformerThreads); } @Override public IDhApiConfigValue chunkLodConverterThreads() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index c9c98cb98..33f325060 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -802,7 +802,7 @@ public class Config .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); - public static final ConfigEntry numberOfDataConverterThreads = new ConfigEntry.Builder() + public static final ConfigEntry numberOfDataTransformerThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, ThreadPresetConfigEventHandler.getDataConverterDefaultThreadCount(), Runtime.getRuntime().availableProcessors()) @@ -818,7 +818,7 @@ public class Config + "\n" + THREAD_NOTE) .build(); - public static final ConfigEntry runTimeRatioForDataConverterThreads = new ConfigEntry.Builder() + public static final ConfigEntry runTimeRatioForDataTransformerThreads = new ConfigEntry.Builder() .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getDataConverterDefaultRunTimeRatio(), 1.0) .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index 5bdbc3722..7d4916c2c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -88,7 +88,7 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan public static int getDataConverterDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions dataConverterThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads, + private final ConfigEntryWithPresetOptions dataTransformerThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfDataTransformerThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 1); @@ -98,7 +98,7 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); public static double getDataConverterDefaultRunTimeRatio() { return 0.25; } - private final ConfigEntryWithPresetOptions dataConverterRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForDataConverterThreads, + private final ConfigEntryWithPresetOptions dataTransformerRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForDataTransformerThreads, new HashMap() {{ this.put(EThreadPreset.MINIMAL_IMPACT, 0.1); @@ -149,8 +149,8 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.configList.add(this.fileHandlerThreadCount); this.configList.add(this.fileHandlerRunTime); - this.configList.add(this.dataConverterThreadCount); - this.configList.add(this.dataConverterRunTime); + this.configList.add(this.dataTransformerThreadCount); + this.configList.add(this.dataTransformerRunTime); this.configList.add(this.chunkLodConverterThreadCount); this.configList.add(this.chunkLodConverterRunTime); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java index 6f3802754..42e76529a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/DataRenderTransformer.java @@ -77,14 +77,15 @@ public class DataRenderTransformer // static setup if (configListener == null) { - configListener = new ConfigChangeListener<>(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads, (threadCount) -> { setThreadPoolSize(threadCount); }); + configListener = new ConfigChangeListener<>(Config.Client.Advanced.MultiThreading.numberOfDataTransformerThreads, (threadCount) -> { setThreadPoolSize(threadCount); }); } + // TODO this didn't seem to be re-sizing when changed via the config if (transformerThreadPool == null || transformerThreadPool.isTerminated()) { LOGGER.info("Starting "+DataRenderTransformer.class.getSimpleName()); - setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfDataConverterThreads.get()); + setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfDataTransformerThreads.get()); } } public static void setThreadPoolSize(int threadPoolSize) @@ -95,7 +96,7 @@ public class DataRenderTransformer transformerThreadPool.shutdown(); } - transformerThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "Full/Render Data Transformer", Config.Client.Advanced.MultiThreading.runTimeRatioForDataConverterThreads); + transformerThreadPool = ThreadUtil.makeRateLimitedThreadPool(threadPoolSize, "Full/Render Data Transformer", Config.Client.Advanced.MultiThreading.runTimeRatioForDataTransformerThreads); } /** diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 3ee414af4..923222373 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -323,12 +323,12 @@ "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForFileHandlerThreads": "Runtime % for file handler threads", - "distanthorizons.config.client.advanced.multiThreading.numberOfDataConverterThreads": - "NO. of data converter threads", - "distanthorizons.config.client.advanced.multiThreading.numberOfDataConverterThreads.@tooltip": + "distanthorizons.config.client.advanced.multiThreading.numberOfDataTransformerThreads": + "NO. of data transformer threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfDataTransformerThreads.@tooltip": "The number of threads used when converting ID data to render-able data. \n(This generally happens when generating new terrain or changing graphics settings). \nCan only be between 1 and your CPU's processor count.", - "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForDataConverterThreads": - "Runtime % for data converter threads", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForDataTransformerThreads": + "Runtime % for data transformer threads", "distanthorizons.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads": "NO. of chunk LOD converter threads", From b71b9e13a6f3960ea902aa6cce79ae8d5efbf052 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 17 Jul 2023 21:43:59 -0500 Subject: [PATCH 20/20] Fix lang for debug menu and allowUnsafeValues --- .../assets/distanthorizons/lang/en_us.json | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 923222373..f55ca4754 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -363,7 +363,11 @@ "distanthorizons.config.client.advanced.debugging.debugWireframeRendering": "Enable Debug Wireframe Rendering", "distanthorizons.config.client.advanced.debugging.debugWireframeRendering.@tooltip": - "If enabled, various wireframes for debugging internal functions will be drawn.", + "If enabled, various wireframes for debugging internal functions will be drawn.", + "distanthorizons.config.client.advanced.debugging.allowUnsafeValues": + "Allow Unsafe UI Values", + "distanthorizons.config.client.advanced.debugging.allowUnsafeValues.@tooltip": + "If enabled, very limited config input validation will be performed. \n\nWarning: enabling this can cause instability or crashing, use at your own risk. \nNote: this is option isn't saved between sessions.", "distanthorizons.config.client.advanced.buffers": "Buffers", @@ -415,36 +419,36 @@ - "distanthorizons.config.client.advanced.debugging.debugConfigScreen": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen": "Debug Config Screen", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.debugConfigScreenNote": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.debugConfigScreenNote": "This screen is to debug features of the config screen", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.boolTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.boolTest": "Boolean test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.byteTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.byteTest": "Byte test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.intTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.intTest": "Integer test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.doubleTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.doubleTest": "Double test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.shortTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.shortTest": "Short test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.longTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.longTest": "Long test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.floatTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.floatTest": "Float test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.stringTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.stringTest": "String test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.listTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.listTest": "List (ArrayList) test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.mapTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.mapTest": "Map (HashMap) test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.categoryTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.categoryTest": "Category test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.linkableTest": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.linkableTest": "Linkable test", - "distanthorizons.config.client.advanced.debugging.debugConfigScreen.linkableTest.@tooltip": + "distanthorizons.config.client.advanced.debugging.exampleConfigScreen.linkableTest.@tooltip": "The value of this should be the same as in Category Test",