wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://gitlab.com/jeseibel/distant-horizons-core

Browse Source
This commit is contained in:
s809
2023-12-18 19:17:14 +05:00
parent 904c1a7d71
commit ff8b1f24d0
7 changed files with 179 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/java/com/seibel/distanthorizons/api/DhApi.java
+16 -1
View File
@@ -52,7 +52,7 @@ public class DhApi
{
/**
* If you can see this Java Doc, this can be ignored. <br>
* This is just to you know that Javadocs are available and that you should use the API jar
* This is just to let you know that Javadocs are available and that you should use the API jar
* instead of the full mod jar. <br><br>
*
* Note: Don't use this string in your code. It may change and is only for reference.
@@ -118,7 +118,11 @@ public class DhApi
//==================//
// always available //
//==================//
// interfaces //
/**
* Used to bind/unbind Distant Horizons Api events.
@@ -139,6 +143,8 @@ public class DhApi
public static final IOverrideInjector<IDhApiOverrideable> overrides = OverrideInjector.INSTANCE;
// getters //
/**
* This version should only be updated when breaking changes are introduced to the Distant Horizons API.
* @since API 1.0.0
@@ -174,4 +180,13 @@ public class DhApi
*/
public static int getNetworkProtocolVersion() { return ModInfo.PROTOCOL_VERSION; }
// methods //
/**
* Returns true if the thread this method was called from is owned by Distant Horizons.
* @since API 1.1.0
*/
public static boolean isDhThread() { return Thread.currentThread().getName().startsWith(ModInfo.THREAD_NAME_PREFIX); }
}
api/src/main/java/com/seibel/distanthorizons/coreapi/ModInfo.java
+2
View File
@@ -48,5 +48,7 @@ public final class ModInfo
public static final String NETWORKING_RESOURCE_NAMESPACE = "distant_horizons";
public static final String MULTIVERSE_PLUGIN_NAMESPACE = "world_control";
/** All DH owned threads should start with this string to allow for easier debugging and profiling. */
public static String THREAD_NAME_PREFIX = "DH-";
}
core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java
+56 -20
View File
@@ -25,8 +25,10 @@ import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.generation.DhLightingEngine;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.Pair;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import com.seibel.distanthorizons.core.world.*;
@@ -52,14 +54,18 @@ public class SharedApi
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final IMinecraftRenderWrapper MC_RENDER = SingletonInjector.INSTANCE.get(IMinecraftRenderWrapper.class);
private static final Set<DhChunkPos> UPDATING_CHUNK_SET = ConcurrentHashMap.newKeySet();
private static final int MAX_UPDATING_CHUNK_COUNT_PER_THREAD = 100;
/** how many chunks can be queued for updating per thread, used to prevent updates from infinitely pilling up if the user flys around extremely fast */
private static final int MAX_UPDATING_CHUNK_COUNT_PER_THREAD = 500;
private static final int MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE = 5_000;
private static final Timer CHUNK_UPDATE_TIMER = new Timer();
private static AbstractDhWorld currentWorld;
private static int lastWorldGenTickDelta = 0;
private static long lastOverloadedLogMessageMsTime = 0;
private static final Timer CHUNK_UPDATE_TIMER = new Timer();
public F3Screen.DynamicMessage f3Message;
@@ -67,7 +73,14 @@ public class SharedApi
// constructor //
//=============//
private SharedApi() { }
private SharedApi()
{
this.f3Message = new F3Screen.DynamicMessage(() ->
{
int maxUpdateCount = MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get();
return LodUtil.formatLog("Queued chunk updates: " + UPDATING_CHUNK_SET.size() + " / " + maxUpdateCount);
});
}
public static void init() { Initializer.init(); }
@@ -148,6 +161,10 @@ public class SharedApi
public void applyChunkUpdate(IChunkWrapper chunkWrapper, ILevelWrapper level, boolean updateNeighborChunks)
{
//========================//
// world and level checks //
//========================//
if (chunkWrapper == null)
{
// shouldn't happen, but just in case
@@ -183,7 +200,42 @@ public class SharedApi
}
// update the necessary chunk(s)
//=====================//
// task limiting check //
//=====================//
int currentQueueCount = UPDATING_CHUNK_SET.size();
int maxQueueCount = MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get();
if (currentQueueCount >= maxQueueCount)
{
// The maximum number of chunks are already queued, don't add more.
// This is done to prevent overloading the system if the user fly's extremely fast and queues too many chunks
long msBetweenLastLog = System.currentTimeMillis() - lastOverloadedLogMessageMsTime;
if (msBetweenLastLog >= MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE)
{
lastOverloadedLogMessageMsTime = System.currentTimeMillis();
LOGGER.warn("Too many chunks queued for updating, max queue count ["+maxQueueCount+"] (["+MAX_UPDATING_CHUNK_COUNT_PER_THREAD+"] per thread). Some LODs may not be updated or may be missing. Please move through the world slower, decrease your vanilla render distance, or increase the CPU load config.");
}
return;
}
// prevent duplicate update requests
if (UPDATING_CHUNK_SET.contains(chunkWrapper.getChunkPos()))
{
// this chunk is already being updated
return;
}
UPDATING_CHUNK_SET.add(chunkWrapper.getChunkPos());
//===============================//
// update the necessary chunk(s) //
//===============================//
if (!updateNeighborChunks)
{
// only update the center chunk
@@ -228,22 +280,6 @@ public class SharedApi
}
private static void bakeChunkLightingAndSendToLevelAsync(IChunkWrapper chunkWrapper, @Nullable ArrayList<IChunkWrapper> neighbourChunkList, IDhLevel dhLevel)
{
if (UPDATING_CHUNK_SET.size() >= MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get())
{
// The maximum number of chunks are already queued, don't add more.
// This is done to prevent overloading the system if the user flys extremely fast and queues too many chunks
return;
}
// prevent duplicate update requests
if (UPDATING_CHUNK_SET.contains(chunkWrapper.getChunkPos()))
{
// this chunk is already being updated
return;
}
UPDATING_CHUNK_SET.add(chunkWrapper.getChunkPos());
// lighting the chunk needs to be done on a separate thread to prevent lagging any of the event threads
ThreadPoolExecutor executor = ThreadPools.getLightPopulatorExecutor();
if (executor == null)
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java
+2 -2
View File
@@ -36,8 +36,8 @@ public interface IIncompleteFullDataSource extends IFullDataSource
{
DhSectionPos inputPos = inputSource.getSectionPos();
DhSectionPos thisPos = this.getSectionPos();
LodUtil.assertTrue(inputPos.getDetailLevel() < thisPos.getDetailLevel());
LodUtil.assertTrue(inputPos.overlapsExactly(this.getSectionPos()), "input source at pos: "+inputPos+" doesn't overlap with this source's pos: "+thisPos);
LodUtil.assertTrue(inputPos.getDetailLevel() < thisPos.getDetailLevel(), "input data source at pos: ["+inputPos+"] has a lower detail level than this: ["+thisPos+"].");
LodUtil.assertTrue(inputPos.overlapsExactly(this.getSectionPos()), "input source at pos: ["+inputPos+"] (converted to ["+inputPos.convertNewToDetailLevel(thisPos.getDetailLevel())+"]) doesn't overlap with this source's pos: ["+thisPos+"].");
if (inputSource.isEmpty())
{
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java
+8 -3
View File
@@ -412,6 +412,10 @@ public class FullDataFileHandler implements IFullDataSourceProvider
return;
}
// can happen if data source caching isn't working correctly
LodUtil.assertTrue(existingFile.pos.equals(existingFullDataSource.getSectionPos()), "Data source returned the wrong position, pooled data source: ["+usePooledDataSources+"]. Expected: ["+existingFile.pos+"] actual: ["+existingFullDataSource.getSectionPos()+"].");
if (showFullDataFileSampling)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
@@ -429,10 +433,11 @@ public class FullDataFileHandler implements IFullDataSourceProvider
//throw e;
}
// pooling temporary data sources massively reduces garbage collector overhead when just sampling (going from ~8 GB/sec to ~90 MB/sec)
if (usePooledDataSources && !existingFile.cacheLoadingDataSource)
// return the pooled data source if necessary
if (usePooledDataSources)
{
existingFile.clearCachedDataSource();
// pooling temporary data sources massively reduces garbage collector overhead when just sampling (going from ~8 GB/sec to ~90 MB/sec)
// get the data loader
AbstractFullDataSourceLoader dataSourceLoader;
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java
+93 -69
View File
@@ -45,7 +45,6 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.AtomicsUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
@@ -54,8 +53,6 @@ import org.apache.logging.log4j.Logger;
/** Represents a File that contains a {@link IFullDataSource}. */
public class FullDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable
{
public static final String FILE_SUFFIX = ".lod";
private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName());
// === Object lifetime tracking ===
@@ -89,8 +86,11 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
* This makes null checks simpler.
*/
private DataSourceReferenceTracker.FullDataSourceSoftRef cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this,null);
private final AtomicReference<CompletableFuture<IFullDataSource>> dataSourceLoadFutureRef = new AtomicReference<>(null);
public volatile Boolean cacheLoadingDataSource = null;
// two different load futures are used to
// prevent accidentally returning a pooled (non-cached) data source
private final AtomicReference<CompletableFuture<IFullDataSource>> cachedDataSourceLoadFutureRef = new AtomicReference<>(null);
private final AtomicReference<CompletableFuture<IFullDataSource>> pooledDataSourceLoadFutureRef = new AtomicReference<>(null);
// === Concurrent Write tracking ===
private final AtomicReference<GuardedMultiAppendQueue> writeQueueRef = new AtomicReference<>(new GuardedMultiAppendQueue());
@@ -171,7 +171,6 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
{
this.cachedFullDataSourceRef.close();
this.cachedFullDataSourceRef.clear();
this.cacheLoadingDataSource = null;
}
return dataExists;
@@ -181,18 +180,44 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
public CompletableFuture<IFullDataSource> getDataSourceWithoutCachingAsync() { return this.getOrLoadCachedDataSourceAsync(false); }
public CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync() { return this.getOrLoadCachedDataSourceAsync(true); }
private CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync(boolean cacheLoadingSource)
/**
* Synchronized to help prevent issues where multiple threads try to read as cached and un-cached at the same time.
* Hopefully isn't necessary and could potentially be removed in the future.
*/
private synchronized CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync(boolean cacheLoadingSource)
{
checkAndLogPhantomDataSourceLifeCycles();
CompletableFuture<IFullDataSource> potentialLoadFuture = this.getCachedDataSourceAsync();
AtomicReference<CompletableFuture<IFullDataSource>> dataSourceLoadFutureRef = cacheLoadingSource ? this.cachedDataSourceLoadFutureRef : this.pooledDataSourceLoadFutureRef;
//========================//
// use the pre-existing //
// load future if present //
//========================//
CompletableFuture<IFullDataSource> preExistingLoadFuture = dataSourceLoadFutureRef.get();
if (preExistingLoadFuture != null)
{
return preExistingLoadFuture;
}
//========================//
// attempt to get the //
// cached data if present //
//========================//
CompletableFuture<IFullDataSource> potentialLoadFuture = null;
if (cacheLoadingSource)
{
potentialLoadFuture = this.getCachedDataSourceAndUpdateIfNeededAsync();
}
if (potentialLoadFuture != null)
{
if (cacheLoadingSource)
{
this.cacheLoadingDataSource = true;
}
// return the in-process future
return potentialLoadFuture;
}
@@ -200,14 +225,14 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
{
// there is no cached data, we'll have to load it
// create a new load future if necessary
potentialLoadFuture = new CompletableFuture<>();
if (!this.dataSourceLoadFutureRef.compareAndSet(null, potentialLoadFuture))
if (!dataSourceLoadFutureRef.compareAndSet(null, potentialLoadFuture))
{
// two threads attempted to start this job at the same time, only use the first future
potentialLoadFuture = this.dataSourceLoadFutureRef.get();
// (shouldn't happen since this method is synchronized, but just in case)
potentialLoadFuture = dataSourceLoadFutureRef.get();
}
this.cacheLoadingDataSource = cacheLoadingSource;
}
@@ -215,7 +240,10 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
final CompletableFuture<IFullDataSource> dataSourceLoadFuture = potentialLoadFuture;
if (!this.doesDtoExist)
{
// create a new DTO and data source
//==================//
// create a new DTO //
// and data source //
//==================//
this.fullDataSourceProvider.onDataFileCreatedAsync(this)
.thenApply((fullDataSource) ->
@@ -229,16 +257,18 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
return fullDataSource;
})
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource))
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource))
.thenAccept((fullDataSource) ->
{
dataSourceLoadFuture.complete(fullDataSource);
this.dataSourceLoadFutureRef.set(null);
dataSourceLoadFutureRef.set(null);
});
}
else
{
// load the existing Meta file and data source
//=========================//
// load the data from file //
//=========================//
if (this.baseMetaData == null)
{
@@ -252,65 +282,54 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
// load the data source
CompletableFuture.supplyAsync(() ->
{
// Load the file.
IFullDataSource fullDataSource;
try (InputStream inputStream = this.getInputStream();
DhDataInputStream compressedStream = new DhDataInputStream(inputStream))
{
// Load the file.
IFullDataSource fullDataSource;
try (InputStream inputStream = this.getInputStream();
DhDataInputStream compressedStream = new DhDataInputStream(inputStream))
if (cacheLoadingSource)
{
if (cacheLoadingSource)
{
fullDataSource = this.fullDataSourceLoader.loadDataSource(this, compressedStream, this.level);
}
else
{
fullDataSource = this.fullDataSourceLoader.loadTemporaryDataSource(this, compressedStream, this.level);
}
fullDataSource = this.fullDataSourceLoader.loadDataSource(this, compressedStream, this.level);
}
catch (Exception ex)
else
{
LOGGER.error("Full Data Load error: "+ ex.getMessage(), ex);
dataSourceLoadFuture.completeExceptionally(ex);
this.dataSourceLoadFutureRef.set(null);
// can happen if there is a missing file or the file was incorrectly formatted, or terminated early
throw new CompletionException(ex);
fullDataSource = this.fullDataSourceLoader.loadTemporaryDataSource(this, compressedStream, this.level);
}
return fullDataSource;
}, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource))
.thenAccept((fullDataSource) ->
}
catch (Exception ex)
{
dataSourceLoadFuture.complete(fullDataSource);
this.dataSourceLoadFutureRef.set(null);
});
LOGGER.error("Full Data Load error: "+ ex.getMessage(), ex);
dataSourceLoadFuture.completeExceptionally(ex);
dataSourceLoadFutureRef.set(null);
// can happen if there is a missing file or the file was incorrectly formatted, or terminated early
throw new CompletionException(ex);
}
return fullDataSource;
}, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource))
.thenAccept((fullDataSource) ->
{
dataSourceLoadFuture.complete(fullDataSource);
dataSourceLoadFutureRef.set(null);
});
}
else
{
// don't load anything if the provider has been shut down
dataSourceLoadFuture.complete(null);
this.dataSourceLoadFutureRef.set(null);
dataSourceLoadFutureRef.set(null);
return dataSourceLoadFuture;
}
}
return dataSourceLoadFuture;
}
/** @return returns null if {@link FullDataMetaFile#cachedFullDataSourceRef} is empty and no cached {@link IFullDataSource} exists. */
private CompletableFuture<IFullDataSource> getCachedDataSourceAsync()
private CompletableFuture<IFullDataSource> getCachedDataSourceAndUpdateIfNeededAsync()
{
// this data source is being written to, use the existing future
CompletableFuture<IFullDataSource> dataSourceLoadFuture = this.dataSourceLoadFutureRef.get();
if (dataSourceLoadFuture != null)
{
return dataSourceLoadFuture;
}
// attempt to get the cached data source
IFullDataSource cachedFullDataSource = this.cachedFullDataSourceRef.get();
if (cachedFullDataSource == null)
@@ -334,7 +353,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
// Create a new future if one doesn't already exist
CompletableFuture<IFullDataSource> newFuture = new CompletableFuture<>();
CompletableFuture<IFullDataSource> oldFuture = AtomicsUtil.compareAndExchange(this.dataSourceLoadFutureRef, null, newFuture);
CompletableFuture<IFullDataSource> oldFuture = AtomicsUtil.compareAndExchange(this.cachedDataSourceLoadFutureRef, null, newFuture);
if (oldFuture != null)
{
@@ -349,17 +368,17 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
// wait for the update to finish before returning the data source
CompletableFuture.supplyAsync(() -> cachedFullDataSource, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource))
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, true))
.thenAccept((fullDataSource) ->
{
newFuture.complete(fullDataSource);
this.dataSourceLoadFutureRef.set(null);
this.cachedDataSourceLoadFutureRef.set(null);
});
}
else
{
// don't update anything if the provider has been shut down
this.dataSourceLoadFutureRef.set(null);
this.cachedDataSourceLoadFutureRef.set(null);
newFuture.complete(null);
}
@@ -499,7 +518,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
color = Color.YELLOW;
}
}
else if (this.dataSourceLoadFutureRef.get() != null)
else if (this.cachedDataSourceLoadFutureRef.get() != null)
{
color = Color.BLUE;
}
@@ -534,7 +553,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
* and stores the result in {@link FullDataMetaFile#cachedFullDataSourceRef}.
*/
@SuppressWarnings("resource") // due to DataObjTracker and DataObjSoftTracker being created outside a try-catch block
private CompletableFuture<IFullDataSource> applyWriteQueueAndSaveAsync(IFullDataSource fullDataSourceToUpdate)
private CompletableFuture<IFullDataSource> applyWriteQueueAndSaveAsync(IFullDataSource fullDataSourceToUpdate, boolean cacheLoadingSource)
{
CompletableFuture<IFullDataSource> completionFuture = new CompletableFuture<>();
@@ -588,8 +607,13 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
}
if (this.cacheLoadingDataSource)
if (cacheLoadingSource)
{
if (fullDataSource != null)
{
LodUtil.assertTrue(this.pos.equals(fullDataSource.getSectionPos()), "Attempting to cache a datasource with the wrong position. Meta file pos: [" + this.pos + "], data source pos: [" + fullDataSource.getSectionPos() + "].");
}
// save the updated data source
this.cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this, fullDataSource);
}
@@ -601,7 +625,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
if (this.needsUpdate)
{
// another update was requested while this update was being processed
if (this.cacheLoadingDataSource)
if (cacheLoadingSource)
{
this.getOrLoadCachedDataSourceAsync();
}
core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java
+2 -2
View File
@@ -24,6 +24,7 @@ import com.seibel.distanthorizons.core.config.types.ConfigEntry;
import com.seibel.distanthorizons.core.util.threading.DhThreadFactory;
import com.seibel.distanthorizons.core.util.threading.RateLimitedThreadPoolExecutor;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import com.seibel.distanthorizons.coreapi.ModInfo;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -38,8 +39,7 @@ public class ThreadUtil
{
private static final Logger LOGGER = LogManager.getLogger();
/** The prefix isn't strictly required, but makes debugging and profiling much easier. */
public static String THREAD_NAME_PREFIX = "DH-";
public static String THREAD_NAME_PREFIX = ModInfo.THREAD_NAME_PREFIX;
public static int MINIMUM_RELATIVE_PRIORITY = -4;
public static int DEFAULT_RELATIVE_PRIORITY = 0;