wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a thread config for ChunkToLodBuilder

Browse Source
This commit is contained in:
James Seibel
2023-06-14 19:42:34 -05:00
parent b2196448f7
commit d502fd4daa
5 changed files with 204 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/lod/core/config/Config.java
+13
View File
@@ -805,6 +805,19 @@ public class Config
+ THREAD_NOTE)
.build();
public static final ConfigEntry<Integer> numberOfChunkLodConverterThreads = new ConfigEntry.Builder<Integer>()
.setMinDefaultMax(1,
Runtime.getRuntime().availableProcessors()/16,
Runtime.getRuntime().availableProcessors())
.comment(""
+ "How many threads should be used to convert Minecraft chunks into LOD data? \n"
+ "\n"
+ "These threads run both when terrain is generated and when\n"
+ "chunks are loaded, unloaded, and modified. \n"
+ "\n"
+ THREAD_NOTE)
.build();
}
public static class GpuBuffers
core/src/main/java/com/seibel/lod/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java
+10
View File
@@ -55,6 +55,15 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan
this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2));
this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0));
}});
private final ConfigEntryWithPresetOptions<EThreadPreset, Integer> chunkLodConverters = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads,
new HashMap<EThreadPreset, Integer>()
{{
this.put(EThreadPreset.MINIMAL_IMPACT, 1);
this.put(EThreadPreset.LOW_IMPACT, getThreadCountByPercent(0.1));
this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.2));
this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4));
this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0));
}});
@@ -70,6 +79,7 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan
this.configList.add(this.bufferBuilders);
this.configList.add(this.fileHandlers);
this.configList.add(this.dataConverters);
this.configList.add(this.chunkLodConverters);
for (ConfigEntryWithPresetOptions<EThreadPreset, ?> config : this.configList)
core/src/main/java/com/seibel/lod/core/dataObjects/transformers/ChunkToLodBuilder.java
+171 -94
View File
@@ -1,8 +1,11 @@
package com.seibel.lod.core.dataObjects.transformers;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import com.seibel.lod.core.config.listeners.ConfigChangeListener;
import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.lod.core.config.Config;
import com.seibel.lod.core.logging.ConfigBasedLogger;
@@ -13,66 +16,65 @@ import com.seibel.lod.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import com.seibel.lod.core.dependencyInjection.SingletonInjector;
import org.apache.logging.log4j.LogManager;
//FIXME: To-Be-Used class
public class ChunkToLodBuilder
public class ChunkToLodBuilder implements AutoCloseable
{
public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(), () -> Config.Client.Advanced.Logging.logLodBuilderEvent.get());
public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(), () -> Config.Client.Advanced.Logging.logLodBuilderEvent.get());
private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
public static final long MAX_TICK_TIME_NS = 1000000000L / 20L;
public static final int THREAD_COUNT = 1;
public static final long MAX_TICK_TIME_NS = 1000000000L / 20L;
private static class Task
private final ConcurrentHashMap<DhChunkPos, IChunkWrapper> latestChunkToBuild = new ConcurrentHashMap<>();
private final ConcurrentLinkedDeque<Task> taskToBuild = new ConcurrentLinkedDeque<>();
private final AtomicInteger runningCount = new AtomicInteger(0);
private int threadCount = -1;
private ExecutorService executorThreadPool = null;
private ConfigChangeListener<Integer> configListener;
//==============//
// constructors //
//==============//
public ChunkToLodBuilder() { this.setupExecutorService(); }
//=================//
// data generation //
//=================//
public CompletableFuture<ChunkSizedFullDataAccessor> tryGenerateData(IChunkWrapper chunkWrapper)
{
final DhChunkPos chunkPos;
final CompletableFuture<ChunkSizedFullDataAccessor> future;
Task(DhChunkPos chunkPos, CompletableFuture<ChunkSizedFullDataAccessor> future)
{
this.chunkPos = chunkPos;
this.future = future;
}
}
private final ConcurrentHashMap<DhChunkPos, IChunkWrapper> latestChunkToBuild = new ConcurrentHashMap<>();
private final ConcurrentLinkedDeque<Task> taskToBuild = new ConcurrentLinkedDeque<>();
private final ExecutorService executor = ThreadUtil.makeThreadPool(THREAD_COUNT, ChunkToLodBuilder.class);
private final AtomicInteger runningCount = new AtomicInteger(0);
public ChunkToLodBuilder() { }
public CompletableFuture<ChunkSizedFullDataAccessor> tryGenerateData(IChunkWrapper chunkWrapper)
{
if (chunkWrapper == null)
if (chunkWrapper == null)
{
throw new NullPointerException("ChunkWrapper cannot be null!");
}
IChunkWrapper oldChunk = this.latestChunkToBuild.put(chunkWrapper.getChunkPos(), chunkWrapper); // an Exchange operation
// If there's old chunk, that means we just replaced an unprocessed old request on generating data on this pos.
// if so, we can just return null to signal this, as the old request's future will instead be the proper one
// that will return the latest generated data.
if (oldChunk != null)
IChunkWrapper oldChunk = this.latestChunkToBuild.put(chunkWrapper.getChunkPos(), chunkWrapper); // an Exchange operation
// If there's old chunk, that means we just replaced an unprocessed old request on generating data on this pos.
// if so, we can just return null to signal this, as the old request's future will instead be the proper one
// that will return the latest generated data.
if (oldChunk != null)
{
return null;
}
// Otherwise, it means we're the first to do so. Let's submit our task to this entry.
CompletableFuture<ChunkSizedFullDataAccessor> future = new CompletableFuture<>();
// Otherwise, it means we're the first to do so. Let's submit our task to this entry.
CompletableFuture<ChunkSizedFullDataAccessor> future = new CompletableFuture<>();
this.taskToBuild.addLast(new Task(chunkWrapper.getChunkPos(), future));
return future;
}
return future;
}
public void tick()
public void tick()
{
if (this.runningCount.get() >= THREAD_COUNT)
if (this.runningCount.get() >= this.threadCount)
{
return;
}
else if (this.taskToBuild.isEmpty())
else if (this.taskToBuild.isEmpty())
{
return;
}
@@ -86,95 +88,94 @@ public class ChunkToLodBuilder
}
for (int i = 0; i<THREAD_COUNT; i++)
for (int i = 0; i< this.threadCount; i++)
{
this.runningCount.incrementAndGet();
CompletableFuture.runAsync(() ->
CompletableFuture.runAsync(() ->
{
try
try
{
_tick();
}
this._tick();
}
finally
{
this.runningCount.decrementAndGet();
}
}, this.executor);
}
}
private void _tick()
}
}, this.executorThreadPool);
}
}
private void _tick()
{
long time = System.nanoTime();
int count = 0;
boolean allDone = false;
while (true)
long time = System.nanoTime();
int count = 0;
boolean allDone = false;
while (true)
{
// run until we either run out of time, or all tasks are complete
if (System.nanoTime() - time > MAX_TICK_TIME_NS && !this.taskToBuild.isEmpty())
if (System.nanoTime() - time > MAX_TICK_TIME_NS && !this.taskToBuild.isEmpty())
{
break;
}
Task task = this.taskToBuild.pollFirst();
if (task == null)
Task task = this.taskToBuild.pollFirst();
if (task == null)
{
allDone = true;
break;
}
allDone = true;
break;
}
count++;
IChunkWrapper latestChunk = this.latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation
if (latestChunk == null)
count++;
IChunkWrapper latestChunk = this.latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation
if (latestChunk == null)
{
LOGGER.error("Somehow Task at "+task.chunkPos+" has latestChunk as null. Skipping task.");
task.future.complete(null);
continue;
}
LOGGER.error("Somehow Task at "+task.chunkPos+" has latestChunk as null. Skipping task.");
task.future.complete(null);
continue;
}
try
try
{
if (LodDataBuilder.canGenerateLodFromChunk(latestChunk))
if (LodDataBuilder.canGenerateLodFromChunk(latestChunk))
{
ChunkSizedFullDataAccessor data = LodDataBuilder.createChunkData(latestChunk);
if (data != null)
ChunkSizedFullDataAccessor data = LodDataBuilder.createChunkData(latestChunk);
if (data != null)
{
task.future.complete(data);
continue;
}
}
}
task.future.complete(data);
continue;
}
}
}
catch (Exception ex)
{
LOGGER.error("Error while processing Task at "+task.chunkPos, ex);
}
LOGGER.error("Error while processing Task at "+task.chunkPos, ex);
}
// Failed to build due to chunk not meeting requirement.
IChunkWrapper casChunk = this.latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null
if (casChunk == null || latestChunk.isStillValid()) // That means CAS have been successful
// Failed to build due to chunk not meeting requirement.
IChunkWrapper casChunk = this.latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null
if (casChunk == null || latestChunk.isStillValid()) // That means CAS have been successful
{
this.taskToBuild.addLast(task); // Then add back the same old task.
}
else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task.
else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task.
{
task.future.complete(null);
}
count--;
}
count--;
}
long time2 = System.nanoTime();
if (!allDone)
long time2 = System.nanoTime();
if (!allDone)
{
//LOGGER.info("Completed {} tasks in {} in this tick", count, Duration.ofNanos(time2 - time));
}
//LOGGER.info("Completed {} tasks in {} in this tick", count, Duration.ofNanos(time2 - time));
}
else if (count > 0)
{
//LOGGER.info("Completed all {} tasks in {}", count, Duration.ofNanos(time2 - time));
}
}
//LOGGER.info("Completed all {} tasks in {}", count, Duration.ofNanos(time2 - time));
}
}
/**
/**
* should be called whenever changing levels/worlds
* to prevent trying to generate LODs for chunk(s) that are no longer loaded
* (which can cause exceptions)
@@ -185,4 +186,80 @@ public class ChunkToLodBuilder
this.latestChunkToBuild.clear();
}
//==========================//
// executor handler methods //
//==========================//
/**
* Creates a new executor. <br>
* Does nothing if an executor already exists.
*/
public void setupExecutorService()
{
// static setup
if (this.configListener == null)
{
this.configListener = new ConfigChangeListener<>(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads, (threadCount) -> { this.setThreadPoolSize(threadCount); });
}
if (this.executorThreadPool == null || this.executorThreadPool.isTerminated())
{
LOGGER.info("Starting "+ChunkToLodBuilder.class.getSimpleName());
this.setThreadPoolSize(Config.Client.Advanced.MultiThreading.numberOfChunkLodConverterThreads.get());
}
}
public void setThreadPoolSize(int threadPoolSize)
{
this.threadCount = threadPoolSize;
this.executorThreadPool = ThreadUtil.makeThreadPool(threadPoolSize, ChunkToLodBuilder.class);
}
/**
* Stops any executing tasks and destroys the executor. <br>
* Does nothing if the executor isn't running.
*/
public void shutdownExecutorService()
{
if (this.executorThreadPool != null)
{
LOGGER.info("Stopping "+ChunkToLodBuilder.class.getSimpleName());
this.executorThreadPool.shutdownNow();
}
}
//==============//
// base methods //
//==============//
@Override
public void close()
{
this.shutdownExecutorService();
this.clearCurrentTasks();
}
//================//
// helper classes //
//================//
private static class Task
{
final DhChunkPos chunkPos;
final CompletableFuture<ChunkSizedFullDataAccessor> future;
Task(DhChunkPos chunkPos, CompletableFuture<ChunkSizedFullDataAccessor> future)
{
this.chunkPos = chunkPos;
this.future = future;
}
}
}
core/src/main/java/com/seibel/lod/core/level/AbstractDhClientLevel.java
+1 -1
View File
@@ -218,7 +218,7 @@ public abstract class AbstractDhClientLevel implements IDhClientLevel
this.fullDataFileHandler.close();
// clear the chunk builder to prevent generating LODs for chunks that are unloaded
this.chunkToLodBuilder.clearCurrentTasks();
this.chunkToLodBuilder.close();
// shutdown the renderer
core/src/main/resources/assets/lod/lang/en_us.json
+9 -5
View File
@@ -308,15 +308,19 @@
"lod.config.client.advanced.multiThreading.numberOfBufferBuilderThreads":
"NO. of buffer builder threads",
"lod.config.client.advanced.multiThreading.numberOfBufferBuilderThreads.@tooltip":
"The number of threads used when building geometry data.\nCan only be between 1 and your CPU's processor count.",
"The number of threads used when building geometry data. \nCan only be between 1 and your CPU's processor count.",
"lod.config.client.advanced.multiThreading.numberOfFileHandlerThreads":
"NO. of file handler threads",
"NO. of file handler threads",
"lod.config.client.advanced.multiThreading.numberOfFileHandlerThreads.@tooltip":
"The number of threads used when building vertex buffers\n(The things sent to your GPU to draw the LODs).\nCan only be between 1 and your CPU's processor count.",
"The number of threads used when building vertex buffers \n(The things sent to your GPU to draw the LODs). \nCan only be between 1 and your CPU's processor count.",
"lod.config.client.advanced.multiThreading.numberOfDataConverterThreads":
"NO. of data converter threads",
"NO. of data converter threads",
"lod.config.client.advanced.multiThreading.numberOfDataConverterThreads.@tooltip":
"The number of threads used when converting ID data to render-able data.\n(This generally happens when generating new terrain or changing graphics settings).\nCan only be between 1 and your CPU's processor count.",
"The number of threads used when converting ID data to render-able data. \n(This generally happens when generating new terrain or changing graphics settings). \nCan only be between 1 and your CPU's processor count.",
"lod.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads":
"NO. of chunk LOD converter threads",
"lod.config.client.advanced.multiThreading.numberOfChunkLodConverterThreads.@tooltip":
"How many threads should be used to convert Minecraft chunks into LOD data? \nThese threads run both when terrain is generated and when \nchunks are loaded, unloaded, and modified.",