From 2c6f2717f01612faad6d02705abcc931a3f2dba4 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 27 Sep 2023 07:53:32 -0500 Subject: [PATCH] Fix partially generated sections not queuing world gen tasks (Most of the time) --- .../fullDatafile/FullDataFileHandler.java | 2 +- .../GeneratedFullDataFileHandler.java | 175 +++++++++++------- .../fullDatafile/IFullDataSourceProvider.java | 3 + .../file/renderfile/RenderDataMetaFile.java | 3 + .../generation/IWorldGenerationQueue.java | 2 + .../core/generation/WorldGenerationQueue.java | 10 +- 6 files changed, 125 insertions(+), 70 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index fd2236665..d580cc217 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -54,7 +54,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected static ExecutorService fileHandlerThreadPool; protected static ConfigChangeListener configListener; - private final ConcurrentHashMap loadedMetaFileBySectionPos = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap loadedMetaFileBySectionPos = new ConcurrentHashMap<>(); protected final IDhLevel level; protected final File saveDir; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 28a343e33..291415cbc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -57,26 +57,78 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler + //===========// + // overrides // + //===========// + + @Override + public CompletableFuture readAsync(DhSectionPos pos) + { + CompletableFuture future = super.readAsync(pos); + return future.thenApply((dataSource) -> + { + // add world gen tasks for missing columns in the data source + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos); + if (worldGenQueue != null && metaFile != null) + { + this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource); + } + + return dataSource; + }); + } + + @Override + public void onRenderDataFileLoaded(DhSectionPos pos) + { + // add world gen tasks for missing columns in the data source + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, false); + if (worldGenQueue != null && metaFile != null) + { + metaFile.getDataSourceWithoutCachingAsync().thenApply((fullDataSource) -> + { + this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, fullDataSource); + return fullDataSource; + }); + } + } + + + //==================// // generation queue // //==================// - /** Assumes there isn't a pre-existing queue. */ - public void setGenerationQueue(IWorldGenerationQueue newWorldGenQueue) + /** + * Assigns the queue for handling world gen and does first time setup as well.
+ * Assumes there isn't a pre-existing queue. + */ { boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); - LOGGER.info("Set world gen queue for level {} to start.", this.level); - this.ForEachFile(metaFile -> { - IFullDataSource data = metaFile.getCachedDataSourceNowOrNull(); - if (data instanceof CompleteFullDataSource) return; - metaFile.genQueueChecked = false; // unset it so it can be checked again - if (data != null) + LOGGER.info("Set world gen queue for level "+this.level+" to start."); + + this.ForEachFile(metaFile -> + { + IFullDataSource dataSource = metaFile.getCachedDataSourceNowOrNull(); + if (dataSource == null) { - metaFile.markNeedsUpdate(); + return; } + + metaFile.genQueueChecked = false; // allow the system to check for missing positions again + this.queueWorldGenForMissingColumnsInDataSource(this.worldGenQueueRef.get(), metaFile, dataSource); + + if (dataSource instanceof CompleteFullDataSource) + { + return; + } + metaFile.markNeedsUpdate(); }); - flushAndSave(); // Trigger an update to the meta files + + this.flushAndSave(); // Trigger an update to the meta files } public void clearGenerationQueue() @@ -96,6 +148,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler }); } + + //=================// // event listeners // //=================// @@ -114,60 +168,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler return newSource; } + + //========// // events // //========// - @Nullable - private CompletableFuture tryStartGenTask(FullDataMetaFile file, IIncompleteFullDataSource dataSource) // TODO after generation is finished, save and free any full datasources that aren't in use (IE high detail ones below the top) - { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - // breaks down the missing positions into the desired detail level that the gen queue could accept - if (worldGenQueue != null && !file.genQueueChecked) - { - DhSectionPos pos = file.pos; - file.genQueueChecked = true; - byte maxSectDataDetailLevel = worldGenQueue.largestDataDetail(); - byte targetDataDetailLevel = dataSource.getDataDetailLevel(); - - if (targetDataDetailLevel > maxSectDataDetailLevel) - { - ArrayList existingFiles = new ArrayList<>(); - byte sectDetailLevel = (byte) (DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + maxSectDataDetailLevel); - pos.forEachChildAtLevel(sectDetailLevel, childPos -> existingFiles.add(this.getLoadOrMakeFile(childPos, true))); - return this.sampleFromFileArray(dataSource, existingFiles, true).thenApply(this::tryPromoteDataSource) - .exceptionally((ex) -> - { - FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, file, ex); - return null; - }); - } - else - { - this.incompleteDataSources.put(pos, dataSource); - // queue this section to be generated - GenTask genTask = new GenTask(pos, new WeakReference<>(dataSource)); - worldGenQueue.submitGenTask(pos, dataSource.getDataDetailLevel(), genTask) - .whenComplete((genTaskResult, ex) -> - { - if (genTaskResult.success) - { - this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos); - this.fireOnGenPosSuccessListeners(pos); - } - else - { - file.genQueueChecked = false; - } - this.incompleteDataSources.remove(pos); - }); - } - // return the empty dataSource (it will be populated later) - return CompletableFuture.completedFuture(dataSource); - } - return null; - } - // Try update the gen queue on this data source. If null, then nothing was done. @Nullable private CompletableFuture updateFromExistingDataSourcesAsync(FullDataMetaFile file, IIncompleteFullDataSource data, boolean usePooledDataSources) @@ -195,6 +201,17 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler }); } } + @Nullable + private CompletableFuture tryStartGenTask(FullDataMetaFile metaFile, IIncompleteFullDataSource dataSource) // TODO after generation is finished, save and free any full datasources that aren't in use (IE high detail ones below the top) + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue != null) + { + this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource); + return CompletableFuture.completedFuture(dataSource); + } + return null; + } @Override public CompletableFuture onDataFileCreatedAsync(FullDataMetaFile file) @@ -305,6 +322,38 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler + //================// + // helper methods // + //================// + + private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, FullDataMetaFile metaFile, IFullDataSource dataSource) + { + // Due to a bug in the current system, some Complete data sources aren't actually complete + // and will need additional generation to finish + //if (dataSource instanceof CompleteFullDataSource) + //{ + // return; + //} + + if (metaFile.genQueueChecked) + { + // world gen has already been checked for this file + return; + } + metaFile.genQueueChecked = true; + + byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.smallestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + ArrayList genPosList = dataSource.getUngeneratedPosList(minGeneratorSectionDetailLevel, true); + + for (DhSectionPos genPos : genPosList) + { + GenTask genTask = new GenTask(dataSource.getSectionPos(), new WeakReference<>(dataSource)); + worldGenQueue.submitGenTask(genPos, dataSource.getSectionPos().getDetailLevel(), genTask); + } + } + + + //================// // helper classes // //================// @@ -329,11 +378,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler @Override - public boolean isMemoryAddressValid() - { - IFullDataSource ref = this.targetFullDataSourceRef.get(); - return ref != null && !((IIncompleteFullDataSource) ref).hasBeenPromoted(); - } + public boolean isMemoryAddressValid() { return this.targetFullDataSourceRef.get() != null; } @Override public Consumer getChunkDataConsumer() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index 679b6ed64..9d8c9f94d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -41,6 +41,9 @@ public interface IFullDataSourceProvider extends AutoCloseable CompletableFuture onDataFileCreatedAsync(FullDataMetaFile file); default CompletableFuture onDataFileUpdateAsync(IFullDataSource fullDataSource, FullDataMetaFile file, boolean dataChanged) { return CompletableFuture.completedFuture(new DataFileUpdateResult(fullDataSource, dataChanged)); } + /** Can be used to update world gen queues or run any other data checking necessary when initially loading a file */ + default void onRenderDataFileLoaded(DhSectionPos pos) { } + File computeDataFilePath(DhSectionPos pos); ExecutorService getIOExecutor(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java index 71ab713a8..b374d7af2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java @@ -125,6 +125,9 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements LodUtil.assertTrue(this.baseMetaData != null); this.doesFileExist = this.file.exists(); DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus); + + // handles world gen queuing for missing columns + this.fullDataSourceProvider.onRenderDataFileLoaded(this.baseMetaData.pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java index 352082edd..c1df7c170 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java @@ -31,6 +31,8 @@ public interface IWorldGenerationQueue extends Closeable { /** the largest numerical detail level */ byte largestDataDetail(); + /** the smallest numerical detail level */ + byte smallestDataDetail(); CompletableFuture submitGenTask(DhSectionPos pos, byte requiredDataDetail, IWorldGenTaskTracker tracker); void cancelGenTasks(Iterable positions); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 6e67176b4..92b8f1947 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -70,8 +70,12 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender public final byte largestDataDetail; @Override public byte largestDataDetail() { return this.largestDataDetail; } + /** lowest numerical detail level allowed */ public final byte smallestDataDetail; + @Override + public byte smallestDataDetail() { return this.smallestDataDetail; } + /** If not null this generator is in the process of shutting down */ private volatile CompletableFuture generatorClosingFuture = null; @@ -302,7 +306,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender // the newly selected task, we cannot use it, // as some chunks may have already been written into. - LOGGER.warn("A task already exists for this position, todo: {}", closestTask.pos); + LOGGER.trace("A task already exists for this position, todo: "+closestTask.pos); } // a task has been started @@ -348,9 +352,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender if (this.alreadyGeneratedPosHashSet.containsKey(inProgressTaskGroup.group.pos)) { // temporary solution to prevent generating the same section multiple times - LOGGER.warn("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping..."); - - //StackTraceElement[] stackTrace = this.alreadyGeneratedPosHashSet.get(inProgressTaskGroup.group.pos); + LOGGER.trace("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping..."); // sending a success result is necessary to make sure the render sections are reloaded correctly inProgressTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateSuccess(new DhSectionPos(granularity, taskPos.getX(), taskPos.getZ()))));