diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index 889462ca6..f625f50b0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -24,7 +24,9 @@ import com.seibel.distanthorizons.core.file.ISourceProvider; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; +import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; import java.util.concurrent.CompletableFuture; /** @@ -36,16 +38,40 @@ public interface IFullDataSourceProvider extends ISourceProvider getAsync(DhSectionPos pos); NewFullDataSource get(DhSectionPos pos); - /** - * If this provider has the ability to create (world gen) or get (networking) - * missing data sources this method will queue the given position - * for generation or retrieval. - */ - void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos); - CompletableFuture updateDataSourceAsync(NewFullDataSource chunkData); /** @return -1 if this provider never has unsaved data sources */ default int getUnsavedDataSourceCount() { return -1; } + + + // retrieval (world gen) // + + /** + * If true this {@link IFullDataSourceProvider} can generate or retrieve + * {@link NewFullDataSource}'s that aren't currently in the database. + */ + default boolean canRetrieveMissingDataSources() { return false; } + + /** @return null if it was unable to generate any positions, an empty array if all positions were generated */ + @Nullable + default ArrayList getPositionsToRetrieve(DhSectionPos pos) { return null; } + /** + * Returns how many positions could potentially be generated for this position assuming the position is empty. + * Used when estimating the total number of retrieval requests. + */ + default int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos) { return -1; } + + /** @return true if the position was queued, false if not */ + default boolean queuePositionForRetrieval(DhSectionPos genPos) { return false; } + /** + * @return false if the provider isn't accepting new requests, + * this can be due to having a full queue or some other + * limiting factor. + */ + default boolean canQueueRetrieval() { return false; } + + /** Can be used to display how many total retrieval requests might be available. */ + default void setTotalRetrievalPositionCount(int newCount) { } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 38d075e89..5cb74f015 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -127,7 +127,7 @@ public class NewFullDataFileHandler @Override protected NewFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) { - // TODO maybe just set children update flags to true? + // TODO maybe just set children update flags to true? return NewFullDataSource.createEmpty(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java index 6adcf1360..81652fa0f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java @@ -31,13 +31,15 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; +import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import org.apache.logging.log4j.Logger; import java.awt.*; import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; @@ -46,18 +48,14 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); + public static final int MAX_WORLD_GEN_REQUESTS_PER_THREAD = 20; + + private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); private final ArrayList onWorldGenTaskCompleteListeners = new ArrayList<>(); protected final DelayedFullDataSourceSaveCache delayedFullDataSourceSaveCache = new DelayedFullDataSourceSaveCache(this::onDataSourceSave, 5_000); - // TODO name better - // this is just the list of section pos that have had their world generation - // calculated and queued this session. - @Deprecated - private final Set genHandledPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); - //=============// @@ -68,29 +66,6 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl - //===========// - // overrides // - //===========// - - @Override - public NewFullDataSource get(DhSectionPos pos) { return this.get(pos, true); } - public NewFullDataSource get(DhSectionPos pos, boolean runWorldGenCheck) - { - NewFullDataSource dataSource = super.get(pos); - - if (runWorldGenCheck) - { - this.tryQueueSection(pos); - } - - return dataSource; - } - - @Override - public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) { this.tryQueueSection(pos); } - - - //==================// // generation queue // //==================// @@ -106,11 +81,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl LOGGER.info("Set world gen queue for level ["+this.level+"]."); } - public void clearGenerationQueue() - { - this.worldGenQueueRef.set(null); - this.genHandledPosSet.clear(); - } + public void clearGenerationQueue() { this.worldGenQueueRef.set(null); } /** Can be used to remove positions that are outside the player's render distance. */ public void removeGenRequestIf(Function removeIf) @@ -122,14 +93,6 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl { worldGenQueue.removeGenRequestIf(removeIf); } - - this.genHandledPosSet.forEach((pos) -> - { - if (removeIf.apply(pos)) - { - this.genHandledPosSet.remove(pos); - } - }); } @Override @@ -191,61 +154,92 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl - //================// - // helper methods // - //================// + //===================================// + // world gen (data source retrieval) // + //===================================// - /** does nothing if this section or one of it's parents has already been queued */ - public void tryQueueSection(DhSectionPos pos) + @Override + public boolean canRetrieveMissingDataSources() { return true; } + + @Override + public void setTotalRetrievalPositionCount(int newCount) { - IWorldGenerationQueue tempWorldGenQueue = this.worldGenQueueRef.get(); - if (tempWorldGenQueue == null) + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue != null) { - return; + worldGenQueue.setEstimatedTotalTaskCount(newCount); } - - if (this.genHandledPosSet.contains(pos)) + } + + @Override + public boolean canQueueRetrieval() + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) { - return; + // we can't queue anything if the world generator isn't set up yet + return false; } - - AtomicBoolean positionAlreadyHandled = new AtomicBoolean(false); - pos.forEachPosUpToDetailLevel(NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL, (parentPos) -> + ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor(); + if (updateExecutor == null || updateExecutor.getQueue().size() >= MAX_UPDATE_TASK_COUNT) { - if (!positionAlreadyHandled.get()) - { - if (this.genHandledPosSet.contains(parentPos)) - { - positionAlreadyHandled.set(true); - } - } - }); - - if (positionAlreadyHandled.get()) - { - return; + // don't queue additional world gen requests if the updater is behind + return false; } - this.genHandledPosSet.add(pos); + // don't queue additional world gen requests beyond the max allotted + int maxQueueCount = MAX_WORLD_GEN_REQUESTS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads.get(); + return worldGenQueue.getWaitingTaskCount() < maxQueueCount; + } + + @Override + public boolean queuePositionForRetrieval(DhSectionPos genPos) + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return false; + } - // get the un-generated pos list - byte minGeneratorSectionDetailLevel = (byte) (tempWorldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + GenTask genTask = new GenTask(genPos); + CompletableFuture worldGenFuture = worldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask); + worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex)); + return true; + } + + @Override + public ArrayList getPositionsToRetrieve(DhSectionPos pos) + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return null; + } + + + // TODO based on the column generation step, only check children that are un-generated + ArrayList generationList = new ArrayList<>(); + byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); pos.forEachChildAtDetailLevel(minGeneratorSectionDetailLevel, (genPos) -> { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue == null) + if (!this.repo.existsWithKey(genPos)) { - return; + // nothing exists for this position, it needs generation + generationList.add(genPos); } - - if (this.repo.existsWithKey(genPos)) + else { - // TODO only pull in the generation steps - NewFullDataSource potentialDataSource = this.get(genPos, false); + byte[] columnGenerationSteps = ((NewFullDataSourceRepo)this.repo).getColumnGenerationStepForPos(genPos); + if (columnGenerationSteps == null) + { + // shouldn't happen, but just in case + return; + } + EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT; checkWorldGenLoop: @@ -254,7 +248,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl for (int z = 0; z < NewFullDataSource.WIDTH; z++) { int index = NewFullDataSource.relativePosToIndex(x, z); - byte genStepValue = potentialDataSource.columnGenerationSteps[index]; + byte genStepValue = columnGenerationSteps[index]; if (genStepValue < currentMinWorldGenStep.value) { @@ -275,19 +269,38 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl if (currentMinWorldGenStep != EDhApiWorldGenerationStep.EMPTY) { - // no world gen needed + // no world gen needed for this position return; } + + generationList.add(genPos); } - - - // queue each new gen task - GenTask genTask = new GenTask(genPos); - CompletableFuture worldGenFuture = tempWorldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask); - worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex)); }); + + return generationList; } + @Override + public int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos) + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return -1; + } + + int minGeneratorSectionDetailLevel = worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + int detailLevelDiff = pos.getDetailLevel() - minGeneratorSectionDetailLevel; + + return BitShiftUtil.powerOfTwo(detailLevelDiff); + } + + + + //=======// + // debug // + //=======// + @Override public void debugRender(DebugRenderer renderer) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 1be730830..c2b4a06c6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -47,6 +47,7 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) + { + // TODO once the legacy data provider has been replaced this can be removed + this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); + return CompletableFuture.completedFuture(null); + } + + //=========// // F3 menu // @@ -148,12 +158,6 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) - { - this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); - return CompletableFuture.completedFuture(null); - } //=====================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java index 52caedbef..cdea7be81 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java @@ -45,6 +45,10 @@ public interface IWorldGenerationQueue extends Closeable int getWaitingTaskCount(); int getInProgressTaskCount(); + /** used for rendering to the F3 menu */ + int getEstimatedTotalTaskCount(); + void setEstimatedTotalTaskCount(int newEstimate); + CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning); void close(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 27cce7d41..cd8c87f08 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -67,13 +67,8 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender /** largest numerical detail level allowed */ public final byte lowestDataDetail; - @Override - public byte lowestDataDetail() { return this.lowestDataDetail; } - /** smallest numerical detail level allowed */ public final byte highestDataDetail; - @Override - public byte highestDataDetail() { return this.highestDataDetail; } /** If not null this generator is in the process of shutting down */ @@ -96,6 +91,9 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender private final HashMap alreadyGeneratedPosHashSet = new HashMap<>(MAX_ALREADY_GENERATED_COUNT); private final Queue alreadyGeneratedPosQueue = new LinkedList<>(); + /** just used for rendering to the F3 menu */ + private int estimatedTotalTaskCount = 0; + //==============// @@ -498,13 +496,24 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender - //=========// - // getters // - //=========// + //===================// + // getters / setters // + //===================// public int getWaitingTaskCount() { return this.waitingTasks.size(); } public int getInProgressTaskCount() { return this.inProgressGenTasksByLodPos.size(); } + @Override + public byte lowestDataDetail() { return this.lowestDataDetail; } + @Override + public byte highestDataDetail() { return this.highestDataDetail; } + + @Override + public int getEstimatedTotalTaskCount() { return this.estimatedTotalTaskCount; } + @Override + public void setEstimatedTotalTaskCount(int newEstimate) { this.estimatedTotalTaskCount = newEstimate; } + + //==========// // shutdown // diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index e49129a71..c1648dab7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -316,7 +316,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get() * LodUtil.CHUNK_WIDTH * 2, // initial position is (0,0) just in case the player hasn't loaded in yet, the tree will be moved once the level starts ticking 0, 0, - this.renderSourceFileHandler); + this.renderSourceFileHandler.fullDataSourceProvider, this.renderSourceFileHandler); RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree); this.renderer = new LodRenderer(renderBufferHandler); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index f008660b1..0d7f5bf98 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -53,8 +53,9 @@ public class WorldGenModule implements Closeable { int waitingCount = worldGenState.worldGenerationQueue.getWaitingTaskCount(); int inProgressCount = worldGenState.worldGenerationQueue.getInProgressTaskCount(); + int totalCountEstimate = worldGenState.worldGenerationQueue.getEstimatedTotalTaskCount(); - return "World Gen Tasks: "+waitingCount+", (in progress: "+inProgressCount+")"; + return "World Gen Tasks: "+waitingCount+"/"+totalCountEstimate+", (in progress: "+inProgressCount+")"; } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index c6cc3a0ab..d64bfe83c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -224,6 +224,12 @@ public class DhSectionPos return (centerBlockPos * BitShiftUtil.powerOfTwo(this.detailLevel)) + positionOffset; } + public int getManhattanBlockDistance(DhBlockPos2D blockPos) + { + return Math.abs(this.getCenterBlockPosX() - blockPos.x) + + Math.abs(this.getCenterBlockPosZ() - blockPos.z); + } + //==================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 5ed35e536..a2c145f24 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -23,20 +23,25 @@ import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree; import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import java.util.ArrayList; import java.util.Iterator; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; /** @@ -48,8 +53,13 @@ public class LodQuadTree extends QuadTree implements AutoClose public static final byte TREE_LOWEST_DETAIL_LEVEL = ColumnRenderSource.SECTION_SIZE_OFFSET; private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + /** there should only ever be one {@link LodQuadTree} so having the thread static should be fine */ + private static final ThreadPoolExecutor FULL_DATA_RETRIEVAL_QUEUE_THREAD = ThreadUtil.makeSingleThreadPool("QuadTree Full Data Retrieval Queue Populator"); + private static final int WORLD_GEN_QUEUE_UPDATE_DELAY_IN_MS = 1_000; + public final int blockRenderDistanceDiameter; + private final IFullDataSourceProvider fullDataSourceProvider; private final IRenderSourceProvider renderSourceProvider; /** @@ -60,6 +70,7 @@ public class LodQuadTree extends QuadTree implements AutoClose private final IDhClientLevel level; //FIXME: Proper hierarchy to remove this reference! private final ConfigChangeListener horizontalScaleChangeListener; private final ReentrantLock treeReadWriteLock = new ReentrantLock(); + private final AtomicBoolean fullDataRetrievalQueueRunning = new AtomicBoolean(false); /** the smallest numerical detail level number that can be rendered */ private byte maxRenderDetailLevel; @@ -80,12 +91,14 @@ public class LodQuadTree extends QuadTree implements AutoClose public LodQuadTree( IDhClientLevel level, int viewDiameterInBlocks, int initialPlayerBlockX, int initialPlayerBlockZ, - IRenderSourceProvider provider) + IFullDataSourceProvider fullDataSourceProvider, + IRenderSourceProvider renderSourceProvider) { super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL); this.level = level; - this.renderSourceProvider = provider; + this.fullDataSourceProvider = fullDataSourceProvider; + this.renderSourceProvider = renderSourceProvider; this.blockRenderDistanceDiameter = viewDiameterInBlocks; this.horizontalScaleChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, (newHorizontalScale) -> this.onHorizontalQualityChange()); @@ -163,6 +176,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // walk through each root node + ArrayList nodesNeedingRetrieval = new ArrayList<>(); Iterator rootPosIterator = this.rootNodePosIterator(); while (rootPosIterator.hasNext()) { @@ -174,14 +188,23 @@ public class LodQuadTree extends QuadTree implements AutoClose } QuadNode rootNode = this.getNode(rootPos); - this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false); + this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false, nodesNeedingRetrieval); + } + + + // full data retrieval (world gen) + if (!this.fullDataRetrievalQueueRunning.get()) + { + this.fullDataRetrievalQueueRunning.set(true); + FULL_DATA_RETRIEVAL_QUEUE_THREAD.execute(() -> this.queueFullDataRetrievalTasks(playerPos, nodesNeedingRetrieval)); } } /** @return whether the current position is able to render (note: not if it IS rendering, just if it is ABLE to.) */ private boolean recursivelyUpdateRenderSectionNode( DhBlockPos2D playerPos, QuadNode rootNode, QuadNode quadNode, DhSectionPos sectionPos, - boolean parentRenderSectionIsEnabled) + boolean parentRenderSectionIsEnabled, + ArrayList nodesNeedingRetrieval) { //===============================// // node and render section setup // @@ -237,7 +260,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled, nodesNeedingRetrieval); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } @@ -259,7 +282,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled, nodesNeedingRetrieval); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } if (!allChildrenSectionsAreLoaded) @@ -276,6 +299,9 @@ public class LodQuadTree extends QuadTree implements AutoClose // TODO this should only equal the expected detail level, the (expectedDetailLevel-1) is a temporary fix to prevent corners from being cut out else if (sectionPos.getDetailLevel() == expectedDetailLevel || sectionPos.getDetailLevel() == expectedDetailLevel - 1) { + // this is the detail level we want to render // + + /* Can be uncommented to easily debug a single render section. */ /* Don't forget the disableRendering() at the bottom though. */ //if (sectionPos.getDetailLevel() == 10 @@ -285,9 +311,9 @@ public class LodQuadTree extends QuadTree implements AutoClose // sectionPos.getZ() == -4 // )) { - // this is the detail level we want to render // // prepare this section for rendering - renderSection.loadRenderSource(this.renderSourceProvider, this.level); // TODO this should fire for the lowest detail level first, wait for it to finish then fire the next highest to prevent waiting forever for 2 million chunk section to finish sampling everything + // TODO this should fire for the lowest detail level first to improve loading speed + renderSection.loadRenderSource(this.renderSourceProvider, this.level); // wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections if (!parentRenderSectionIsEnabled && renderSection.canRenderNow()) @@ -308,6 +334,11 @@ public class LodQuadTree extends QuadTree implements AutoClose }); } } + + if (!renderSection.isFullyGenerated()) + { + nodesNeedingRetrieval.add(renderSection); + } } //else //{ @@ -400,7 +431,7 @@ public class LodQuadTree extends QuadTree implements AutoClose */ public void clearRenderDataCache() { - if (this.treeReadWriteLock.tryLock()) + if (this.treeReadWriteLock.tryLock()) // TODO make async, can lock render thread { try { @@ -451,14 +482,70 @@ public class LodQuadTree extends QuadTree implements AutoClose + //=================================// + // full data retrieval (world gen) // + //=================================// + + private void queueFullDataRetrievalTasks(DhBlockPos2D playerPos, ArrayList nodesNeedingRetrieval) + { + try + { + // add a slight delay since we don't need to check the world gen queue every tick + Thread.sleep(WORLD_GEN_QUEUE_UPDATE_DELAY_IN_MS); + + // sort the nodes from nearest to farthest + nodesNeedingRetrieval.sort((a, b) -> + { + int aDist = a.pos.getManhattanBlockDistance(playerPos); + int bDist = b.pos.getManhattanBlockDistance(playerPos); + return Integer.compare(aDist, bDist); + }); + + // add retrieval tasks to the queue + for (int i = 0; i < nodesNeedingRetrieval.size(); i++) + { + LodRenderSection renderSection = nodesNeedingRetrieval.get(i); + if (!this.fullDataSourceProvider.canQueueRetrieval()) + { + break; + } + + renderSection.tryQueuingMissingLodRetrieval(this.fullDataSourceProvider); + } + + // calculate an estimate for the max number of tasks for the queue + int totalWorldGenCount = 0; + for (int i = 0; i < nodesNeedingRetrieval.size(); i++) + { + LodRenderSection renderSection = nodesNeedingRetrieval.get(i); + if (!renderSection.missingPositionsCalculated()) + { + // may be higher than the actual amount + totalWorldGenCount += this.fullDataSourceProvider.getMaxPossibleRetrievalPositionCountForPos(renderSection.pos); + } + else + { + totalWorldGenCount += renderSection.ungeneratedPositionCount(); + } + } + this.fullDataSourceProvider.setTotalRetrievalPositionCount(totalWorldGenCount); + } + catch (Exception e) + { + LOGGER.error("Unexpected error: "+e.getMessage(), e); + } + finally + { + this.fullDataRetrievalQueueRunning.set(false); + } + } + + //==================// // config listeners // //==================// - private void onHorizontalQualityChange() - { - this.clearRenderDataCache(); - } + private void onHorizontalQualityChange() { this.clearRenderDataCache(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 629fffab0..0a72522de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; import com.seibel.distanthorizons.core.enums.EDhDirection; +import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -36,6 +37,7 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import org.apache.logging.log4j.Logger; import java.awt.*; +import java.util.ArrayList; import java.util.Objects; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; @@ -84,6 +86,10 @@ public class LodRenderSection implements IDebugRenderable private final QuadTree parentQuadTree; + private boolean missingPositionsCalculated = false; + /** should be an empty array if no positions need to be generated */ + private ArrayList missingGenerationPos = null; + //=============// @@ -407,6 +413,56 @@ public class LodRenderSection implements IDebugRenderable + //=================================// + // full data retrieval (world gen) // + //=================================// + + public boolean isFullyGenerated() { return this.missingPositionsCalculated && this.missingGenerationPos.size() == 0; } + public boolean missingPositionsCalculated() { return this.missingPositionsCalculated; } + public int ungeneratedPositionCount() { return (this.missingGenerationPos != null) ? this.missingGenerationPos.size() : 0; } + + public void tryQueuingMissingLodRetrieval(IFullDataSourceProvider fullDataSourceProvider) + { + if (fullDataSourceProvider.canRetrieveMissingDataSources() && fullDataSourceProvider.canQueueRetrieval()) + { + // calculate the missing positions if not already done + if (!this.missingPositionsCalculated) + { + this.missingGenerationPos = fullDataSourceProvider.getPositionsToRetrieve(this.pos); + if (this.missingGenerationPos != null) + { + this.missingPositionsCalculated = true; + } + } + + // if the missing positions were found, queue them + if (this.missingGenerationPos != null) + { + // queue from last to first to prevent shifting the array unnecessarily + for (int i = this.missingGenerationPos.size() - 1; i >= 0; i--) + { + if (!fullDataSourceProvider.canQueueRetrieval()) + { + // the data source provider isn't accepting any more jobs + break; + } + + DhSectionPos pos = this.missingGenerationPos.remove(i); + boolean positionQueued = fullDataSourceProvider.queuePositionForRetrieval(pos); + if (!positionQueued) + { + // shouldn't normally happen, but just in case + this.missingGenerationPos.add(pos); + break; + } + } + } + } + } + + + + //==============// // base methods // //==============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java index 6f0ab5ff0..f5d27003c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java @@ -209,6 +209,26 @@ public class NewFullDataSourceRepo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( + "select ColumnGenerationStep " + + "from "+this.getTableName()+" " + + "WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ()); + + if (resultMap != null) + { + return (byte[]) resultMap.get("ColumnGenerationStep"); + } + else + { + return null; + } + } + }