wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "Merge branch 'main' of https://gitlab.com/jeseibel/distant-horizons-core"

Browse Source 
This reverts commit 522a799516, reversing
changes made to 04e43ebec8.
This commit is contained in:
s809
2023-12-18 22:15:34 +05:00
parent 522a799516
commit a2cc1c2913
7 changed files with 97 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/java/com/seibel/distanthorizons/api/DhApi.java
+1 -16
View File
@@ -52,7 +52,7 @@ public class DhApi
{
/**
* If you can see this Java Doc, this can be ignored. <br>
* This is just to let you know that Javadocs are available and that you should use the API jar
* This is just to you know that Javadocs are available and that you should use the API jar
* instead of the full mod jar. <br><br>
*
* Note: Don't use this string in your code. It may change and is only for reference.
@@ -118,11 +118,7 @@ public class DhApi
//==================//
// always available //
//==================//
// interfaces //
/**
* Used to bind/unbind Distant Horizons Api events.
@@ -143,8 +139,6 @@ public class DhApi
public static final IOverrideInjector<IDhApiOverrideable> overrides = OverrideInjector.INSTANCE;
// getters //
/**
* This version should only be updated when breaking changes are introduced to the Distant Horizons API.
* @since API 1.0.0
@@ -180,13 +174,4 @@ public class DhApi
*/
public static int getNetworkProtocolVersion() { return ModInfo.PROTOCOL_VERSION; }
// methods //
/**
* Returns true if the thread this method was called from is owned by Distant Horizons.
* @since API 1.1.0
*/
public static boolean isDhThread() { return Thread.currentThread().getName().startsWith(ModInfo.THREAD_NAME_PREFIX); }
}
api/src/main/java/com/seibel/distanthorizons/coreapi/ModInfo.java
-2
View File
@@ -48,7 +48,5 @@ public final class ModInfo
public static final String NETWORKING_RESOURCE_NAMESPACE = "distant_horizons";
public static final String MULTIVERSE_PLUGIN_NAMESPACE = "world_control";
/** All DH owned threads should start with this string to allow for easier debugging and profiling. */
public static String THREAD_NAME_PREFIX = "DH-";
}
core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java
+20 -56
View File
@@ -25,10 +25,8 @@ import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.generation.DhLightingEngine;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.Pair;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import com.seibel.distanthorizons.core.world.*;
@@ -54,18 +52,14 @@ public class SharedApi
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final IMinecraftRenderWrapper MC_RENDER = SingletonInjector.INSTANCE.get(IMinecraftRenderWrapper.class);
private static final Set<DhChunkPos> UPDATING_CHUNK_SET = ConcurrentHashMap.newKeySet();
/** how many chunks can be queued for updating per thread, used to prevent updates from infinitely pilling up if the user flys around extremely fast */
private static final int MAX_UPDATING_CHUNK_COUNT_PER_THREAD = 500;
private static final int MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE = 5_000;
private static final int MAX_UPDATING_CHUNK_COUNT_PER_THREAD = 100;
private static final Timer CHUNK_UPDATE_TIMER = new Timer();
private static AbstractDhWorld currentWorld;
private static int lastWorldGenTickDelta = 0;
private static long lastOverloadedLogMessageMsTime = 0;
public F3Screen.DynamicMessage f3Message;
private static final Timer CHUNK_UPDATE_TIMER = new Timer();
@@ -73,14 +67,7 @@ public class SharedApi
// constructor //
//=============//
private SharedApi()
{
this.f3Message = new F3Screen.DynamicMessage(() ->
{
int maxUpdateCount = MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get();
return LodUtil.formatLog("Queued chunk updates: " + UPDATING_CHUNK_SET.size() + " / " + maxUpdateCount);
});
}
private SharedApi() { }
public static void init() { Initializer.init(); }
@@ -161,10 +148,6 @@ public class SharedApi
public void applyChunkUpdate(IChunkWrapper chunkWrapper, ILevelWrapper level, boolean updateNeighborChunks)
{
//========================//
// world and level checks //
//========================//
if (chunkWrapper == null)
{
// shouldn't happen, but just in case
@@ -200,42 +183,7 @@ public class SharedApi
}
//=====================//
// task limiting check //
//=====================//
int currentQueueCount = UPDATING_CHUNK_SET.size();
int maxQueueCount = MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get();
if (currentQueueCount >= maxQueueCount)
{
// The maximum number of chunks are already queued, don't add more.
// This is done to prevent overloading the system if the user fly's extremely fast and queues too many chunks
long msBetweenLastLog = System.currentTimeMillis() - lastOverloadedLogMessageMsTime;
if (msBetweenLastLog >= MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE)
{
lastOverloadedLogMessageMsTime = System.currentTimeMillis();
LOGGER.warn("Too many chunks queued for updating, max queue count ["+maxQueueCount+"] (["+MAX_UPDATING_CHUNK_COUNT_PER_THREAD+"] per thread). Some LODs may not be updated or may be missing. Please move through the world slower, decrease your vanilla render distance, or increase the CPU load config.");
}
return;
}
// prevent duplicate update requests
if (UPDATING_CHUNK_SET.contains(chunkWrapper.getChunkPos()))
{
// this chunk is already being updated
return;
}
UPDATING_CHUNK_SET.add(chunkWrapper.getChunkPos());
//===============================//
// update the necessary chunk(s) //
//===============================//
// update the necessary chunk(s)
if (!updateNeighborChunks)
{
// only update the center chunk
@@ -280,6 +228,22 @@ public class SharedApi
}
private static void bakeChunkLightingAndSendToLevelAsync(IChunkWrapper chunkWrapper, @Nullable ArrayList<IChunkWrapper> neighbourChunkList, IDhLevel dhLevel)
{
if (UPDATING_CHUNK_SET.size() >= MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get())
{
// The maximum number of chunks are already queued, don't add more.
// This is done to prevent overloading the system if the user flys extremely fast and queues too many chunks
return;
}
// prevent duplicate update requests
if (UPDATING_CHUNK_SET.contains(chunkWrapper.getChunkPos()))
{
// this chunk is already being updated
return;
}
UPDATING_CHUNK_SET.add(chunkWrapper.getChunkPos());
// lighting the chunk needs to be done on a separate thread to prevent lagging any of the event threads
ThreadPoolExecutor executor = ThreadPools.getLightPopulatorExecutor();
if (executor == null)
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java
+2 -2
View File
@@ -36,8 +36,8 @@ public interface IIncompleteFullDataSource extends IFullDataSource
{
DhSectionPos inputPos = inputSource.getSectionPos();
DhSectionPos thisPos = this.getSectionPos();
LodUtil.assertTrue(inputPos.getDetailLevel() < thisPos.getDetailLevel(), "input data source at pos: ["+inputPos+"] has a lower detail level than this: ["+thisPos+"].");
LodUtil.assertTrue(inputPos.overlapsExactly(this.getSectionPos()), "input source at pos: ["+inputPos+"] (converted to ["+inputPos.convertNewToDetailLevel(thisPos.getDetailLevel())+"]) doesn't overlap with this source's pos: ["+thisPos+"].");
LodUtil.assertTrue(inputPos.getDetailLevel() < thisPos.getDetailLevel());
LodUtil.assertTrue(inputPos.overlapsExactly(this.getSectionPos()), "input source at pos: "+inputPos+" doesn't overlap with this source's pos: "+thisPos);
if (inputSource.isEmpty())
{
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java
+3 -8
View File
@@ -412,10 +412,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider
return;
}
// can happen if data source caching isn't working correctly
LodUtil.assertTrue(existingFile.pos.equals(existingFullDataSource.getSectionPos()), "Data source returned the wrong position, pooled data source: ["+usePooledDataSources+"]. Expected: ["+existingFile.pos+"] actual: ["+existingFullDataSource.getSectionPos()+"].");
if (showFullDataFileSampling)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
@@ -433,11 +429,10 @@ public class FullDataFileHandler implements IFullDataSourceProvider
//throw e;
}
// return the pooled data source if necessary
if (usePooledDataSources)
// pooling temporary data sources massively reduces garbage collector overhead when just sampling (going from ~8 GB/sec to ~90 MB/sec)
if (usePooledDataSources && !existingFile.cacheLoadingDataSource)
{
// pooling temporary data sources massively reduces garbage collector overhead when just sampling (going from ~8 GB/sec to ~90 MB/sec)
existingFile.clearCachedDataSource();
// get the data loader
AbstractFullDataSourceLoader dataSourceLoader;
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java
+69 -93
View File
@@ -45,6 +45,7 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.AtomicsUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
@@ -53,6 +54,8 @@ import org.apache.logging.log4j.Logger;
/** Represents a File that contains a {@link IFullDataSource}. */
public class FullDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable
{
public static final String FILE_SUFFIX = ".lod";
private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName());
// === Object lifetime tracking ===
@@ -86,11 +89,8 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
* This makes null checks simpler.
*/
private DataSourceReferenceTracker.FullDataSourceSoftRef cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this,null);
// two different load futures are used to
// prevent accidentally returning a pooled (non-cached) data source
private final AtomicReference<CompletableFuture<IFullDataSource>> cachedDataSourceLoadFutureRef = new AtomicReference<>(null);
private final AtomicReference<CompletableFuture<IFullDataSource>> pooledDataSourceLoadFutureRef = new AtomicReference<>(null);
private final AtomicReference<CompletableFuture<IFullDataSource>> dataSourceLoadFutureRef = new AtomicReference<>(null);
public volatile Boolean cacheLoadingDataSource = null;
// === Concurrent Write tracking ===
private final AtomicReference<GuardedMultiAppendQueue> writeQueueRef = new AtomicReference<>(new GuardedMultiAppendQueue());
@@ -171,6 +171,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
{
this.cachedFullDataSourceRef.close();
this.cachedFullDataSourceRef.clear();
this.cacheLoadingDataSource = null;
}
return dataExists;
@@ -180,44 +181,18 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
public CompletableFuture<IFullDataSource> getDataSourceWithoutCachingAsync() { return this.getOrLoadCachedDataSourceAsync(false); }
public CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync() { return this.getOrLoadCachedDataSourceAsync(true); }
/**
* Synchronized to help prevent issues where multiple threads try to read as cached and un-cached at the same time.
* Hopefully isn't necessary and could potentially be removed in the future.
*/
private synchronized CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync(boolean cacheLoadingSource)
private CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync(boolean cacheLoadingSource)
{
checkAndLogPhantomDataSourceLifeCycles();
AtomicReference<CompletableFuture<IFullDataSource>> dataSourceLoadFutureRef = cacheLoadingSource ? this.cachedDataSourceLoadFutureRef : this.pooledDataSourceLoadFutureRef;
//========================//
// use the pre-existing //
// load future if present //
//========================//
CompletableFuture<IFullDataSource> preExistingLoadFuture = dataSourceLoadFutureRef.get();
if (preExistingLoadFuture != null)
{
return preExistingLoadFuture;
}
//========================//
// attempt to get the //
// cached data if present //
//========================//
CompletableFuture<IFullDataSource> potentialLoadFuture = null;
if (cacheLoadingSource)
{
potentialLoadFuture = this.getCachedDataSourceAndUpdateIfNeededAsync();
}
CompletableFuture<IFullDataSource> potentialLoadFuture = this.getCachedDataSourceAsync();
if (potentialLoadFuture != null)
{
if (cacheLoadingSource)
{
this.cacheLoadingDataSource = true;
}
// return the in-process future
return potentialLoadFuture;
}
@@ -225,14 +200,14 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
{
// there is no cached data, we'll have to load it
// create a new load future if necessary
potentialLoadFuture = new CompletableFuture<>();
if (!dataSourceLoadFutureRef.compareAndSet(null, potentialLoadFuture))
if (!this.dataSourceLoadFutureRef.compareAndSet(null, potentialLoadFuture))
{
// two threads attempted to start this job at the same time, only use the first future
// (shouldn't happen since this method is synchronized, but just in case)
potentialLoadFuture = dataSourceLoadFutureRef.get();
potentialLoadFuture = this.dataSourceLoadFutureRef.get();
}
this.cacheLoadingDataSource = cacheLoadingSource;
}
@@ -240,10 +215,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
final CompletableFuture<IFullDataSource> dataSourceLoadFuture = potentialLoadFuture;
if (!this.doesDtoExist)
{
//==================//
// create a new DTO //
// and data source //
//==================//
// create a new DTO and data source
this.fullDataSourceProvider.onDataFileCreatedAsync(this)
.thenApply((fullDataSource) ->
@@ -257,18 +229,16 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
return fullDataSource;
})
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource))
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource))
.thenAccept((fullDataSource) ->
{
dataSourceLoadFuture.complete(fullDataSource);
dataSourceLoadFutureRef.set(null);
this.dataSourceLoadFutureRef.set(null);
});
}
else
{
//=========================//
// load the data from file //
//=========================//
// load the existing Meta file and data source
if (this.baseMetaData == null)
{
@@ -282,54 +252,65 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
// load the data source
CompletableFuture.supplyAsync(() ->
{
// Load the file.
IFullDataSource fullDataSource;
try (InputStream inputStream = this.getInputStream();
DhDataInputStream compressedStream = new DhDataInputStream(inputStream))
{
if (cacheLoadingSource)
// Load the file.
IFullDataSource fullDataSource;
try (InputStream inputStream = this.getInputStream();
DhDataInputStream compressedStream = new DhDataInputStream(inputStream))
{
fullDataSource = this.fullDataSourceLoader.loadDataSource(this, compressedStream, this.level);
if (cacheLoadingSource)
{
fullDataSource = this.fullDataSourceLoader.loadDataSource(this, compressedStream, this.level);
}
else
{
fullDataSource = this.fullDataSourceLoader.loadTemporaryDataSource(this, compressedStream, this.level);
}
}
else
catch (Exception ex)
{
fullDataSource = this.fullDataSourceLoader.loadTemporaryDataSource(this, compressedStream, this.level);
LOGGER.error("Full Data Load error: "+ ex.getMessage(), ex);
dataSourceLoadFuture.completeExceptionally(ex);
this.dataSourceLoadFutureRef.set(null);
// can happen if there is a missing file or the file was incorrectly formatted, or terminated early
throw new CompletionException(ex);
}
}
catch (Exception ex)
return fullDataSource;
}, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource))
.thenAccept((fullDataSource) ->
{
LOGGER.error("Full Data Load error: "+ ex.getMessage(), ex);
dataSourceLoadFuture.completeExceptionally(ex);
dataSourceLoadFutureRef.set(null);
// can happen if there is a missing file or the file was incorrectly formatted, or terminated early
throw new CompletionException(ex);
}
return fullDataSource;
}, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource))
.thenAccept((fullDataSource) ->
{
dataSourceLoadFuture.complete(fullDataSource);
dataSourceLoadFutureRef.set(null);
});
dataSourceLoadFuture.complete(fullDataSource);
this.dataSourceLoadFutureRef.set(null);
});
}
else
{
// don't load anything if the provider has been shut down
dataSourceLoadFuture.complete(null);
dataSourceLoadFutureRef.set(null);
this.dataSourceLoadFutureRef.set(null);
return dataSourceLoadFuture;
}
}
return dataSourceLoadFuture;
}
/** @return returns null if {@link FullDataMetaFile#cachedFullDataSourceRef} is empty and no cached {@link IFullDataSource} exists. */
private CompletableFuture<IFullDataSource> getCachedDataSourceAndUpdateIfNeededAsync()
private CompletableFuture<IFullDataSource> getCachedDataSourceAsync()
{
// this data source is being written to, use the existing future
CompletableFuture<IFullDataSource> dataSourceLoadFuture = this.dataSourceLoadFutureRef.get();
if (dataSourceLoadFuture != null)
{
return dataSourceLoadFuture;
}
// attempt to get the cached data source
IFullDataSource cachedFullDataSource = this.cachedFullDataSourceRef.get();
if (cachedFullDataSource == null)
@@ -353,7 +334,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
// Create a new future if one doesn't already exist
CompletableFuture<IFullDataSource> newFuture = new CompletableFuture<>();
CompletableFuture<IFullDataSource> oldFuture = AtomicsUtil.compareAndExchange(this.cachedDataSourceLoadFutureRef, null, newFuture);
CompletableFuture<IFullDataSource> oldFuture = AtomicsUtil.compareAndExchange(this.dataSourceLoadFutureRef, null, newFuture);
if (oldFuture != null)
{
@@ -368,17 +349,17 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
// wait for the update to finish before returning the data source
CompletableFuture.supplyAsync(() -> cachedFullDataSource, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, true))
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource))
.thenAccept((fullDataSource) ->
{
newFuture.complete(fullDataSource);
this.cachedDataSourceLoadFutureRef.set(null);
this.dataSourceLoadFutureRef.set(null);
});
}
else
{
// don't update anything if the provider has been shut down
this.cachedDataSourceLoadFutureRef.set(null);
this.dataSourceLoadFutureRef.set(null);
newFuture.complete(null);
}
@@ -518,7 +499,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
color = Color.YELLOW;
}
}
else if (this.cachedDataSourceLoadFutureRef.get() != null)
else if (this.dataSourceLoadFutureRef.get() != null)
{
color = Color.BLUE;
}
@@ -553,7 +534,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
* and stores the result in {@link FullDataMetaFile#cachedFullDataSourceRef}.
*/
@SuppressWarnings("resource") // due to DataObjTracker and DataObjSoftTracker being created outside a try-catch block
private CompletableFuture<IFullDataSource> applyWriteQueueAndSaveAsync(IFullDataSource fullDataSourceToUpdate, boolean cacheLoadingSource)
private CompletableFuture<IFullDataSource> applyWriteQueueAndSaveAsync(IFullDataSource fullDataSourceToUpdate)
{
CompletableFuture<IFullDataSource> completionFuture = new CompletableFuture<>();
@@ -607,13 +588,8 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
}
if (cacheLoadingSource)
if (this.cacheLoadingDataSource)
{
if (fullDataSource != null)
{
LodUtil.assertTrue(this.pos.equals(fullDataSource.getSectionPos()), "Attempting to cache a datasource with the wrong position. Meta file pos: [" + this.pos + "], data source pos: [" + fullDataSource.getSectionPos() + "].");
}
// save the updated data source
this.cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this, fullDataSource);
}
@@ -625,7 +601,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
if (this.needsUpdate)
{
// another update was requested while this update was being processed
if (cacheLoadingSource)
if (this.cacheLoadingDataSource)
{
this.getOrLoadCachedDataSourceAsync();
}
core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java
+2 -2
View File
@@ -24,7 +24,6 @@ import com.seibel.distanthorizons.core.config.types.ConfigEntry;
import com.seibel.distanthorizons.core.util.threading.DhThreadFactory;
import com.seibel.distanthorizons.core.util.threading.RateLimitedThreadPoolExecutor;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import com.seibel.distanthorizons.coreapi.ModInfo;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -39,7 +38,8 @@ public class ThreadUtil
{
private static final Logger LOGGER = LogManager.getLogger();
public static String THREAD_NAME_PREFIX = ModInfo.THREAD_NAME_PREFIX;
/** The prefix isn't strictly required, but makes debugging and profiling much easier. */
public static String THREAD_NAME_PREFIX = "DH-";
public static int MINIMUM_RELATIVE_PRIORITY = -4;
public static int DEFAULT_RELATIVE_PRIORITY = 0;