wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix partially generated sections not queuing world gen tasks

Browse Source 
(Most of the time)
This commit is contained in:
James Seibel
2023-09-27 07:53:32 -05:00
parent bf17a72a7a
commit 2c6f2717f0
6 changed files with 125 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java
+1 -1
View File
@@ -54,7 +54,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
protected static ExecutorService fileHandlerThreadPool;
protected static ConfigChangeListener<Integer> configListener;
private final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> loadedMetaFileBySectionPos = new ConcurrentHashMap<>();
protected final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> loadedMetaFileBySectionPos = new ConcurrentHashMap<>();
protected final IDhLevel level;
protected final File saveDir;
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java
+110 -65
View File
@@ -57,26 +57,78 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
//===========//
// overrides //
//===========//
@Override
public CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos)
{
CompletableFuture<IFullDataSource> future = super.readAsync(pos);
return future.thenApply((dataSource) ->
{
// add world gen tasks for missing columns in the data source
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos);
if (worldGenQueue != null && metaFile != null)
{
this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource);
}
return dataSource;
});
}
@Override
public void onRenderDataFileLoaded(DhSectionPos pos)
{
// add world gen tasks for missing columns in the data source
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, false);
if (worldGenQueue != null && metaFile != null)
{
metaFile.getDataSourceWithoutCachingAsync().thenApply((fullDataSource) ->
{
this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, fullDataSource);
return fullDataSource;
});
}
}
//==================//
// generation queue //
//==================//
/** Assumes there isn't a pre-existing queue. */
public void setGenerationQueue(IWorldGenerationQueue newWorldGenQueue)
/**
* Assigns the queue for handling world gen and does first time setup as well. <br>
* Assumes there isn't a pre-existing queue.
*/
{
boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue);
LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!");
LOGGER.info("Set world gen queue for level {} to start.", this.level);
this.ForEachFile(metaFile -> {
IFullDataSource data = metaFile.getCachedDataSourceNowOrNull();
if (data instanceof CompleteFullDataSource) return;
metaFile.genQueueChecked = false; // unset it so it can be checked again
if (data != null)
LOGGER.info("Set world gen queue for level "+this.level+" to start.");
this.ForEachFile(metaFile ->
{
IFullDataSource dataSource = metaFile.getCachedDataSourceNowOrNull();
if (dataSource == null)
{
metaFile.markNeedsUpdate();
return;
}
metaFile.genQueueChecked = false; // allow the system to check for missing positions again
this.queueWorldGenForMissingColumnsInDataSource(this.worldGenQueueRef.get(), metaFile, dataSource);
if (dataSource instanceof CompleteFullDataSource)
{
return;
}
metaFile.markNeedsUpdate();
});
flushAndSave(); // Trigger an update to the meta files
this.flushAndSave(); // Trigger an update to the meta files
}
public void clearGenerationQueue()
@@ -96,6 +148,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
});
}
//=================//
// event listeners //
//=================//
@@ -114,60 +168,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
return newSource;
}
//========//
// events //
//========//
@Nullable
private CompletableFuture<IFullDataSource> tryStartGenTask(FullDataMetaFile file, IIncompleteFullDataSource dataSource) // TODO after generation is finished, save and free any full datasources that aren't in use (IE high detail ones below the top)
{
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
// breaks down the missing positions into the desired detail level that the gen queue could accept
if (worldGenQueue != null && !file.genQueueChecked)
{
DhSectionPos pos = file.pos;
file.genQueueChecked = true;
byte maxSectDataDetailLevel = worldGenQueue.largestDataDetail();
byte targetDataDetailLevel = dataSource.getDataDetailLevel();
if (targetDataDetailLevel > maxSectDataDetailLevel)
{
ArrayList<FullDataMetaFile> existingFiles = new ArrayList<>();
byte sectDetailLevel = (byte) (DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + maxSectDataDetailLevel);
pos.forEachChildAtLevel(sectDetailLevel, childPos -> existingFiles.add(this.getLoadOrMakeFile(childPos, true)));
return this.sampleFromFileArray(dataSource, existingFiles, true).thenApply(this::tryPromoteDataSource)
.exceptionally((ex) ->
{
FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, file, ex);
return null;
});
}
else
{
this.incompleteDataSources.put(pos, dataSource);
// queue this section to be generated
GenTask genTask = new GenTask(pos, new WeakReference<>(dataSource));
worldGenQueue.submitGenTask(pos, dataSource.getDataDetailLevel(), genTask)
.whenComplete((genTaskResult, ex) ->
{
if (genTaskResult.success)
{
this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos);
this.fireOnGenPosSuccessListeners(pos);
}
else
{
file.genQueueChecked = false;
}
this.incompleteDataSources.remove(pos);
});
}
// return the empty dataSource (it will be populated later)
return CompletableFuture.completedFuture(dataSource);
}
return null;
}
// Try update the gen queue on this data source. If null, then nothing was done.
@Nullable
private CompletableFuture<IFullDataSource> updateFromExistingDataSourcesAsync(FullDataMetaFile file, IIncompleteFullDataSource data, boolean usePooledDataSources)
@@ -195,6 +201,17 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
});
}
}
@Nullable
private CompletableFuture<IFullDataSource> tryStartGenTask(FullDataMetaFile metaFile, IIncompleteFullDataSource dataSource) // TODO after generation is finished, save and free any full datasources that aren't in use (IE high detail ones below the top)
{
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
if (worldGenQueue != null)
{
this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource);
return CompletableFuture.completedFuture(dataSource);
}
return null;
}
@Override
public CompletableFuture<IFullDataSource> onDataFileCreatedAsync(FullDataMetaFile file)
@@ -305,6 +322,38 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
//================//
// helper methods //
//================//
private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, FullDataMetaFile metaFile, IFullDataSource dataSource)
{
// Due to a bug in the current system, some Complete data sources aren't actually complete
// and will need additional generation to finish
//if (dataSource instanceof CompleteFullDataSource)
//{
// return;
//}
if (metaFile.genQueueChecked)
{
// world gen has already been checked for this file
return;
}
metaFile.genQueueChecked = true;
byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.smallestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
ArrayList<DhSectionPos> genPosList = dataSource.getUngeneratedPosList(minGeneratorSectionDetailLevel, true);
for (DhSectionPos genPos : genPosList)
{
GenTask genTask = new GenTask(dataSource.getSectionPos(), new WeakReference<>(dataSource));
worldGenQueue.submitGenTask(genPos, dataSource.getSectionPos().getDetailLevel(), genTask);
}
}
//================//
// helper classes //
//================//
@@ -329,11 +378,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
@Override
public boolean isMemoryAddressValid()
{
IFullDataSource ref = this.targetFullDataSourceRef.get();
return ref != null && !((IIncompleteFullDataSource) ref).hasBeenPromoted();
}
public boolean isMemoryAddressValid() { return this.targetFullDataSourceRef.get() != null; }
@Override
public Consumer<ChunkSizedFullDataAccessor> getChunkDataConsumer()
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java
+3
View File
@@ -41,6 +41,9 @@ public interface IFullDataSourceProvider extends AutoCloseable
CompletableFuture<IFullDataSource> onDataFileCreatedAsync(FullDataMetaFile file);
default CompletableFuture<DataFileUpdateResult> onDataFileUpdateAsync(IFullDataSource fullDataSource, FullDataMetaFile file, boolean dataChanged) { return CompletableFuture.completedFuture(new DataFileUpdateResult(fullDataSource, dataChanged)); }
/** Can be used to update world gen queues or run any other data checking necessary when initially loading a file */
default void onRenderDataFileLoaded(DhSectionPos pos) { }
File computeDataFilePath(DhSectionPos pos);
ExecutorService getIOExecutor();
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java
+3
View File
@@ -125,6 +125,9 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
LodUtil.assertTrue(this.baseMetaData != null);
this.doesFileExist = this.file.exists();
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus);
// handles world gen queuing for missing columns
this.fullDataSourceProvider.onRenderDataFileLoaded(this.baseMetaData.pos);
}
core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java
+2
View File
@@ -31,6 +31,8 @@ public interface IWorldGenerationQueue extends Closeable
{
/** the largest numerical detail level */
byte largestDataDetail();
/** the smallest numerical detail level */
byte smallestDataDetail();
CompletableFuture<WorldGenResult> submitGenTask(DhSectionPos pos, byte requiredDataDetail, IWorldGenTaskTracker tracker);
void cancelGenTasks(Iterable<DhSectionPos> positions);
core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java
+6 -4
View File
@@ -70,8 +70,12 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender
public final byte largestDataDetail;
@Override
public byte largestDataDetail() { return this.largestDataDetail; }
/** lowest numerical detail level allowed */
public final byte smallestDataDetail;
@Override
public byte smallestDataDetail() { return this.smallestDataDetail; }
/** If not null this generator is in the process of shutting down */
private volatile CompletableFuture<Void> generatorClosingFuture = null;
@@ -302,7 +306,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender
// the newly selected task, we cannot use it,
// as some chunks may have already been written into.
LOGGER.warn("A task already exists for this position, todo: {}", closestTask.pos);
LOGGER.trace("A task already exists for this position, todo: "+closestTask.pos);
}
// a task has been started
@@ -348,9 +352,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender
if (this.alreadyGeneratedPosHashSet.containsKey(inProgressTaskGroup.group.pos))
{
// temporary solution to prevent generating the same section multiple times
LOGGER.warn("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping...");
//StackTraceElement[] stackTrace = this.alreadyGeneratedPosHashSet.get(inProgressTaskGroup.group.pos);
LOGGER.trace("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping...");
// sending a success result is necessary to make sure the render sections are reloaded correctly
inProgressTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateSuccess(new DhSectionPos(granularity, taskPos.getX(), taskPos.getZ()))));