Fix world gen for extreme render distances

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java

@@ -24,7 +24,9 @@ import com.seibel.distanthorizons.core.file.ISourceProvider;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.sql.repo.FullDataRepo;
+import org.jetbrains.annotations.Nullable;
 
+import java.util.ArrayList;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -36,16 +38,40 @@ public interface IFullDataSourceProvider extends ISourceProvider<NewFullDataSour
 	CompletableFuture<NewFullDataSource> getAsync(DhSectionPos pos);
 	NewFullDataSource get(DhSectionPos pos);
 	
-	/** 
-	 * If this provider has the ability to create (world gen) or get (networking)
-	 * missing data sources this method will queue the given position
-	 * for generation or retrieval.
-	 */
-	void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos);
-	
 	CompletableFuture<Void> updateDataSourceAsync(NewFullDataSource chunkData);
 	
 	/** @return -1 if this provider never has unsaved data sources */
 	default int getUnsavedDataSourceCount() { return -1; }
 	
+	
+	
+	// retrieval (world gen) //
+	
+	/**
+	 * If true this {@link IFullDataSourceProvider} can generate or retrieve
+	 * {@link NewFullDataSource}'s that aren't currently in the database.
+	 */
+	default boolean canRetrieveMissingDataSources() { return false; }
+	
+	/** @return null if it was unable to generate any positions, an empty array if all positions were generated */
+	@Nullable
+	default ArrayList<DhSectionPos> getPositionsToRetrieve(DhSectionPos pos)  { return null; }
+	/**
+	 * Returns how many positions could potentially be generated for this position assuming the position is empty.
+	 * Used when estimating the total number of retrieval requests.
+	 */
+	default int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos)  { return -1; }
+	
+	/** @return true if the position was queued, false if not */
+	default boolean queuePositionForRetrieval(DhSectionPos genPos) { return false; }
+	/** 
+	 * @return false if the provider isn't accepting new requests,
+	 *          this can be due to having a full queue or some other
+	 *          limiting factor.
+	 */
+	default boolean canQueueRetrieval() { return false; }
+	
+	/** Can be used to display how many total retrieval requests might be available. */
+	default void setTotalRetrievalPositionCount(int newCount) {  }
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java

@@ -127,7 +127,7 @@ public class NewFullDataFileHandler
 	@Override
 	protected NewFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos)
 	{
-		// TODO maybe just set children update flags to true?
+		//  TODO maybe just set children update flags to true?
 		return NewFullDataSource.createEmpty(pos);
 	}
 	

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java

@@ -31,13 +31,15 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
 import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
+import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo;
 import com.seibel.distanthorizons.core.util.LodUtil;
+import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
+import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
 import org.apache.logging.log4j.Logger;
 
 import java.awt.*;
 import java.util.*;
 import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -46,18 +48,14 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl
 {
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	
-	private final AtomicReference<IWorldGenerationQueue> worldGenQueueRef = new AtomicReference<>(null);
+	public static final int MAX_WORLD_GEN_REQUESTS_PER_THREAD = 20; 
 	
+	
+	private final AtomicReference<IWorldGenerationQueue> worldGenQueueRef = new AtomicReference<>(null);
 	private final ArrayList<IOnWorldGenCompleteListener> onWorldGenTaskCompleteListeners = new ArrayList<>();
 	
 	protected final DelayedFullDataSourceSaveCache delayedFullDataSourceSaveCache = new DelayedFullDataSourceSaveCache(this::onDataSourceSave, 5_000);
 	
-	// TODO name better
-	//  this is just the list of section pos that have had their world generation
-	//  calculated and queued this session.
-	@Deprecated
-	private final Set<DhSectionPos> genHandledPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>());
-	
 	
 	
 	//=============//
@@ -68,29 +66,6 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl
 	
 	
 	
-	//===========//
-	// overrides //
-	//===========//
-	
-	@Override
-	public NewFullDataSource get(DhSectionPos pos) { return this.get(pos, true); }
-	public NewFullDataSource get(DhSectionPos pos, boolean runWorldGenCheck)
-	{
-		NewFullDataSource dataSource = super.get(pos);
-		
-		if (runWorldGenCheck)
-		{
-			this.tryQueueSection(pos);
-		}
-		
-		return dataSource;
-	}
-	
-	@Override
-	public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) { this.tryQueueSection(pos); }
-	
-	
-	
 	//==================//
 	// generation queue //
 	//==================//
@@ -106,11 +81,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl
 		LOGGER.info("Set world gen queue for level ["+this.level+"].");
 	}
 	
-	public void clearGenerationQueue()
-	{
-		this.worldGenQueueRef.set(null);
-		this.genHandledPosSet.clear();
-	}
+	public void clearGenerationQueue() { this.worldGenQueueRef.set(null); }
 	
 	/** Can be used to remove positions that are outside the player's render distance. */
 	public void removeGenRequestIf(Function<DhSectionPos, Boolean> removeIf)
@@ -122,14 +93,6 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl
 		{
 			worldGenQueue.removeGenRequestIf(removeIf);
 		}
-		
-		this.genHandledPosSet.forEach((pos) ->
-		{
-			if (removeIf.apply(pos))
-			{
-				this.genHandledPosSet.remove(pos);
-			}
-		});
 	}
 	
 	@Override
@@ -191,61 +154,92 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl
 	
 	
 	
-	//================//
-	// helper methods //
-	//================//
+	//===================================//
+	// world gen (data source retrieval) //
+	//===================================//
 	
-	/** does nothing if this section or one of it's parents has already been queued */
-	public void tryQueueSection(DhSectionPos pos)
+	@Override
+	public boolean canRetrieveMissingDataSources() { return true; }
+	
+	@Override
+	public void setTotalRetrievalPositionCount(int newCount) 
 	{
-		IWorldGenerationQueue tempWorldGenQueue = this.worldGenQueueRef.get();
-		if (tempWorldGenQueue == null)
+		IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
+		if (worldGenQueue != null)
 		{
-			return;
+			worldGenQueue.setEstimatedTotalTaskCount(newCount);
 		}
-		
-		if (this.genHandledPosSet.contains(pos))
+	}
+	
+	@Override
+	public boolean canQueueRetrieval()
+	{
+		IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
+		if (worldGenQueue == null)
 		{
-			return;
+			// we can't queue anything if the world generator isn't set up yet
+			return false;
 		}
 		
 		
-		
-		AtomicBoolean positionAlreadyHandled = new AtomicBoolean(false);
-		pos.forEachPosUpToDetailLevel(NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL, (parentPos) ->
+		ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor();
+		if (updateExecutor == null || updateExecutor.getQueue().size() >= MAX_UPDATE_TASK_COUNT)
 		{
-			if (!positionAlreadyHandled.get())
-			{
-				if (this.genHandledPosSet.contains(parentPos))
-				{
-					positionAlreadyHandled.set(true);
-				}
-			}
-		});
-		
-		if (positionAlreadyHandled.get())
-		{
-			return;
+			// don't queue additional world gen requests if the updater is behind
+			return false;
 		}
-		this.genHandledPosSet.add(pos);
 		
 		
+		// don't queue additional world gen requests beyond the max allotted
+		int maxQueueCount = MAX_WORLD_GEN_REQUESTS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads.get();
+		return worldGenQueue.getWaitingTaskCount() < maxQueueCount; 
+	}
+	
+	@Override
+	public boolean queuePositionForRetrieval(DhSectionPos genPos)
+	{
+		IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
+		if (worldGenQueue == null)
+		{
+			return false;
+		}
 		
-		// get the un-generated pos list
-		byte minGeneratorSectionDetailLevel = (byte) (tempWorldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
+		GenTask genTask = new GenTask(genPos);
+		CompletableFuture<WorldGenResult> worldGenFuture = worldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask);
+		worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex));
 		
+		return true;
+	}
+	
+	@Override
+	public ArrayList<DhSectionPos> getPositionsToRetrieve(DhSectionPos pos)
+	{
+		IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
+		if (worldGenQueue == null)
+		{
+			return null;
+		}
+		
+		
+		// TODO based on the column generation step, only check children that are un-generated
+		ArrayList<DhSectionPos> generationList = new ArrayList<>();
+		byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
 		pos.forEachChildAtDetailLevel(minGeneratorSectionDetailLevel, (genPos) ->
 		{
-			IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
-			if (worldGenQueue == null)
+			if (!this.repo.existsWithKey(genPos))
 			{
-				return;
+				// nothing exists for this position, it needs generation
+				generationList.add(genPos);
 			}
-			
-			if (this.repo.existsWithKey(genPos))
+			else
 			{
-				// TODO only pull in the generation steps
-				NewFullDataSource potentialDataSource = this.get(genPos, false);
+				byte[] columnGenerationSteps = ((NewFullDataSourceRepo)this.repo).getColumnGenerationStepForPos(genPos);
+				if (columnGenerationSteps == null)
+				{
+					// shouldn't happen, but just in case
+					return;
+				}
+				
 				
 				EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT;
 				checkWorldGenLoop:
@@ -254,7 +248,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl
 					for (int z = 0; z < NewFullDataSource.WIDTH; z++)
 					{
 						int index = NewFullDataSource.relativePosToIndex(x, z);
-						byte genStepValue = potentialDataSource.columnGenerationSteps[index];
+						byte genStepValue = columnGenerationSteps[index];
 						
 						if (genStepValue < currentMinWorldGenStep.value)
 						{
@@ -275,19 +269,38 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl
 				
 				if (currentMinWorldGenStep != EDhApiWorldGenerationStep.EMPTY)
 				{
-					// no world gen needed
+					// no world gen needed for this position
 					return;
 				}
+				
+				generationList.add(genPos);
 			}
-			
-			
-			// queue each new gen task
-			GenTask genTask = new GenTask(genPos);
-			CompletableFuture<WorldGenResult> worldGenFuture = tempWorldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask);
-			worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex));
 		});
+		
+		return generationList;
 	}
 	
+	@Override
+	public int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos)  
+	{
+		IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
+		if (worldGenQueue == null)
+		{
+			return -1;
+		}
+		
+		int minGeneratorSectionDetailLevel = worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
+		int detailLevelDiff = pos.getDetailLevel() - minGeneratorSectionDetailLevel;
+		
+		return BitShiftUtil.powerOfTwo(detailLevelDiff);
+	}
+	
+	
+	
+	//=======//
+	// debug //
+	//=======//
+	
 	@Override
 	public void debugRender(DebugRenderer renderer)
 	{

core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java

@@ -47,6 +47,7 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler<Col
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	
 	private final F3Screen.NestedMessage threadPoolMsg;
+	private int totalRetrievalPositionCount = 0;
 	
 	public final IFullDataSourceProvider fullDataSourceProvider;
 	
@@ -73,10 +74,6 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler<Col
 	@Override
 	public ColumnRenderSource get(DhSectionPos pos)
 	{
-		// call the full data provider to make sure the full data is up to date
-		// and any necessary world generation has been queued/completed
-		this.fullDataSourceProvider.queuePositionForGenerationOrRetrievalIfNecessary(pos);
-		
 		return super.get(pos);
 	}
 	
@@ -119,6 +116,19 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler<Col
 	
 	
 	
+	//=====================//
+	// extension overrides //
+	//=====================//
+	
+	@Override
+	public CompletableFuture<Void> updateDataSourceAsync(NewFullDataSource inputDataSource)
+	{
+		// TODO once the legacy data provider has been replaced this can be removed
+		this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource);
+		return CompletableFuture.completedFuture(null);
+	}
+	
+	
 	
 	//=========//
 	// F3 menu //
@@ -148,12 +158,6 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler<Col
 		return lines.toArray(new String[0]);
 	}
 	
-	@Override
-	public CompletableFuture<Void> updateDataSourceAsync(NewFullDataSource inputDataSource)
-	{
-		this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource);
-		return CompletableFuture.completedFuture(null);
-	}
 	
 	
 	//=====================//

core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java

@@ -45,6 +45,10 @@ public interface IWorldGenerationQueue extends Closeable
 	int getWaitingTaskCount();
 	int getInProgressTaskCount();
 	
+	/** used for rendering to the F3 menu */
+	int getEstimatedTotalTaskCount();
+	void setEstimatedTotalTaskCount(int newEstimate);
+	
 	CompletableFuture<Void> startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning);
 	void close();
 	

core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java

@@ -67,13 +67,8 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender
 	
 	/** largest numerical detail level allowed */
 	public final byte lowestDataDetail;
-	@Override
-	public byte lowestDataDetail() { return this.lowestDataDetail; }
-	
 	/** smallest numerical detail level allowed */
 	public final byte highestDataDetail;
-	@Override
-	public byte highestDataDetail() { return this.highestDataDetail; }
 	
 	
 	/** If not null this generator is in the process of shutting down */
@@ -96,6 +91,9 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender
 	private final HashMap<DhSectionPos, StackTraceElement[]> alreadyGeneratedPosHashSet = new HashMap<>(MAX_ALREADY_GENERATED_COUNT);
 	private final Queue<DhSectionPos> alreadyGeneratedPosQueue = new LinkedList<>();
 	
+	/** just used for rendering to the F3 menu */
+	private int estimatedTotalTaskCount = 0;
+	
 	
 	
 	//==============//
@@ -498,13 +496,24 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender
 	
 	
 	
-	//=========//
-	// getters //
-	//=========//
+	//===================//
+	// getters / setters //
+	//===================//
 	
 	public int getWaitingTaskCount() { return this.waitingTasks.size(); }
 	public int getInProgressTaskCount() { return this.inProgressGenTasksByLodPos.size(); }
 	
+	@Override
+	public byte lowestDataDetail() { return this.lowestDataDetail; }
+	@Override
+	public byte highestDataDetail() { return this.highestDataDetail; }
+	
+	@Override
+	public int getEstimatedTotalTaskCount() { return this.estimatedTotalTaskCount; }
+	@Override
+	public void setEstimatedTotalTaskCount(int newEstimate) { this.estimatedTotalTaskCount = newEstimate; }
+	
+	
 	
 	//==========//
 	// shutdown //

core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java

@@ -316,7 +316,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
 			this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get() * LodUtil.CHUNK_WIDTH * 2,
 					// initial position is (0,0) just in case the player hasn't loaded in yet, the tree will be moved once the level starts ticking
 					0, 0,
-					this.renderSourceFileHandler);
+					this.renderSourceFileHandler.fullDataSourceProvider, this.renderSourceFileHandler);
 			
 			RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree);
 			this.renderer = new LodRenderer(renderBufferHandler);

core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java

@@ -53,8 +53,9 @@ public class WorldGenModule implements Closeable
 			{
 				int waitingCount = worldGenState.worldGenerationQueue.getWaitingTaskCount();
 				int inProgressCount = worldGenState.worldGenerationQueue.getInProgressTaskCount();
+				int totalCountEstimate = worldGenState.worldGenerationQueue.getEstimatedTotalTaskCount();
 				
-				return "World Gen Tasks: "+waitingCount+", (in progress: "+inProgressCount+")";
+				return "World Gen Tasks: "+waitingCount+"/"+totalCountEstimate+", (in progress: "+inProgressCount+")";
 			}
 			else
 			{

core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java

@@ -224,6 +224,12 @@ public class DhSectionPos
 		return (centerBlockPos * BitShiftUtil.powerOfTwo(this.detailLevel)) + positionOffset;
 	}
 	
+	public int getManhattanBlockDistance(DhBlockPos2D blockPos)
+	{
+		return Math.abs(this.getCenterBlockPosX() - blockPos.x)
+				+ Math.abs(this.getCenterBlockPosZ() - blockPos.z);
+	}
+	
 	
 	
 	//==================//

core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java

@@ -23,20 +23,25 @@ import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality;
 import com.seibel.distanthorizons.core.config.Config;
 import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener;
 import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
+import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
 import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider;
 import com.seibel.distanthorizons.core.level.IDhClientLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.util.LodUtil;
+import com.seibel.distanthorizons.core.util.ThreadUtil;
 import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode;
 import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree;
 import com.seibel.distanthorizons.coreapi.util.MathUtil;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
@@ -48,8 +53,13 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 	public static final byte TREE_LOWEST_DETAIL_LEVEL = ColumnRenderSource.SECTION_SIZE_OFFSET;
 	
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
+	/** there should only ever be one {@link LodQuadTree} so having the thread static should be fine */
+	private static final ThreadPoolExecutor FULL_DATA_RETRIEVAL_QUEUE_THREAD = ThreadUtil.makeSingleThreadPool("QuadTree Full Data Retrieval Queue Populator");
+	private static final int WORLD_GEN_QUEUE_UPDATE_DELAY_IN_MS = 1_000;
+	
 	
 	public final int blockRenderDistanceDiameter;
+	private final IFullDataSourceProvider fullDataSourceProvider;
 	private final IRenderSourceProvider renderSourceProvider;
 	
 	/**
@@ -60,6 +70,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 	private final IDhClientLevel level; //FIXME: Proper hierarchy to remove this reference!
 	private final ConfigChangeListener<EHorizontalQuality> horizontalScaleChangeListener;
 	private final ReentrantLock treeReadWriteLock = new ReentrantLock();
+	private final AtomicBoolean fullDataRetrievalQueueRunning = new AtomicBoolean(false);
 	
 	/** the smallest numerical detail level number that can be rendered */
 	private byte maxRenderDetailLevel;
@@ -80,12 +91,14 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 	public LodQuadTree(
 			IDhClientLevel level, int viewDiameterInBlocks,
 			int initialPlayerBlockX, int initialPlayerBlockZ,
-			IRenderSourceProvider provider)
+			IFullDataSourceProvider fullDataSourceProvider, 
+			IRenderSourceProvider renderSourceProvider)
 	{
 		super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL);
 		
 		this.level = level;
-		this.renderSourceProvider = provider;
+		this.fullDataSourceProvider = fullDataSourceProvider;
+		this.renderSourceProvider = renderSourceProvider;
 		this.blockRenderDistanceDiameter = viewDiameterInBlocks;
 		
 		this.horizontalScaleChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, (newHorizontalScale) -> this.onHorizontalQualityChange());
@@ -163,6 +176,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 		
 		
 		// walk through each root node
+		ArrayList<LodRenderSection> nodesNeedingRetrieval = new ArrayList<>();
 		Iterator<DhSectionPos> rootPosIterator = this.rootNodePosIterator();
 		while (rootPosIterator.hasNext())
 		{
@@ -174,14 +188,23 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 			}
 			
 			QuadNode<LodRenderSection> rootNode = this.getNode(rootPos);
-			this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false);
+			this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false, nodesNeedingRetrieval);
+		}
+		
+		
+		// full data retrieval (world gen)
+		if (!this.fullDataRetrievalQueueRunning.get())
+		{
+			this.fullDataRetrievalQueueRunning.set(true);
+			FULL_DATA_RETRIEVAL_QUEUE_THREAD.execute(() -> this.queueFullDataRetrievalTasks(playerPos, nodesNeedingRetrieval));
 		}
 	}
 	/** @return whether the current position is able to render (note: not if it IS rendering, just if it is ABLE to.) */
 	private boolean recursivelyUpdateRenderSectionNode(
 			DhBlockPos2D playerPos, 
 			QuadNode<LodRenderSection> rootNode, QuadNode<LodRenderSection> quadNode, DhSectionPos sectionPos, 
-			boolean parentRenderSectionIsEnabled)
+			boolean parentRenderSectionIsEnabled,
+			ArrayList<LodRenderSection> nodesNeedingRetrieval)
 	{
 		//===============================//
 		// node and render section setup //
@@ -237,7 +260,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 				DhSectionPos childPos = childPosIterator.next();
 				QuadNode<LodRenderSection> childNode = rootNode.getNode(childPos);
 				
-				boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled);
+				boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled, nodesNeedingRetrieval);
 				allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded;
 			}
 			
@@ -259,7 +282,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 					DhSectionPos childPos = childPosIterator.next();
 					QuadNode<LodRenderSection> childNode = rootNode.getNode(childPos);
 					
-					boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled);
+					boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled, nodesNeedingRetrieval);
 					allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded;
 				}
 				if (!allChildrenSectionsAreLoaded)
@@ -276,6 +299,9 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 		// TODO this should only equal the expected detail level, the (expectedDetailLevel-1) is a temporary fix to prevent corners from being cut out 
 		else if (sectionPos.getDetailLevel() == expectedDetailLevel || sectionPos.getDetailLevel() == expectedDetailLevel - 1)
 		{
+			// this is the detail level we want to render //
+			
+			
 			/* Can be uncommented to easily debug a single render section. */ 
 			/* Don't forget the disableRendering() at the bottom though. */
 			//if (sectionPos.getDetailLevel() == 10
@@ -285,9 +311,9 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 			//			sectionPos.getZ() == -4
 			//	))
 			{
-				// this is the detail level we want to render //
 				// prepare this section for rendering
-				renderSection.loadRenderSource(this.renderSourceProvider, this.level); // TODO this should fire for the lowest detail level first, wait for it to finish then fire the next highest to prevent waiting forever for 2 million chunk section to finish sampling everything
+				// TODO this should fire for the lowest detail level first to improve loading speed
+				renderSection.loadRenderSource(this.renderSourceProvider, this.level);
 				
 				// wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections
 				if (!parentRenderSectionIsEnabled && renderSection.canRenderNow())
@@ -308,6 +334,11 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 						});
 					}
 				}
+				
+				if (!renderSection.isFullyGenerated())
+				{
+					nodesNeedingRetrieval.add(renderSection);
+				}
 			}
 			//else
 			//{
@@ -400,7 +431,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 	 */
 	public void clearRenderDataCache()
 	{
-		if (this.treeReadWriteLock.tryLock())
+		if (this.treeReadWriteLock.tryLock()) // TODO make async, can lock render thread
 		{
 			try
 			{
@@ -451,14 +482,70 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
 	
 	
 	
+	//=================================//
+	// full data retrieval (world gen) //
+	//=================================//
+	
+	private void queueFullDataRetrievalTasks(DhBlockPos2D playerPos, ArrayList<LodRenderSection> nodesNeedingRetrieval)
+	{
+		try
+		{
+			// add a slight delay since we don't need to check the world gen queue every tick
+			Thread.sleep(WORLD_GEN_QUEUE_UPDATE_DELAY_IN_MS);
+			
+			// sort the nodes from nearest to farthest
+			nodesNeedingRetrieval.sort((a, b) ->
+			{
+				int aDist = a.pos.getManhattanBlockDistance(playerPos);
+				int bDist = b.pos.getManhattanBlockDistance(playerPos);
+				return Integer.compare(aDist, bDist);
+			});
+			
+			// add retrieval tasks to the queue
+			for (int i = 0; i < nodesNeedingRetrieval.size(); i++)
+			{
+				LodRenderSection renderSection = nodesNeedingRetrieval.get(i);
+				if (!this.fullDataSourceProvider.canQueueRetrieval())
+				{
+					break;
+				}
+				
+				renderSection.tryQueuingMissingLodRetrieval(this.fullDataSourceProvider);
+			}
+			
+			// calculate an estimate for the max number of tasks for the queue
+			int totalWorldGenCount = 0;
+			for (int i = 0; i < nodesNeedingRetrieval.size(); i++)
+			{
+				LodRenderSection renderSection = nodesNeedingRetrieval.get(i);
+				if (!renderSection.missingPositionsCalculated())
+				{
+					// may be higher than the actual amount
+					totalWorldGenCount += this.fullDataSourceProvider.getMaxPossibleRetrievalPositionCountForPos(renderSection.pos);
+				}
+				else
+				{
+					totalWorldGenCount += renderSection.ungeneratedPositionCount();
+				}
+			}
+			this.fullDataSourceProvider.setTotalRetrievalPositionCount(totalWorldGenCount);
+		}
+		catch (Exception e)
+		{
+			LOGGER.error("Unexpected error: "+e.getMessage(), e);
+		}
+		finally
+		{
+			this.fullDataRetrievalQueueRunning.set(false);
+		}
+	}
+	
+	
 	//==================//
 	// config listeners //
 	//==================//
 	
-	private void onHorizontalQualityChange()
-	{
-		this.clearRenderDataCache();
-	}
+	private void onHorizontalQualityChange() { this.clearRenderDataCache(); }
 	
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java

@@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.config.Config;
 import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
 import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder;
 import com.seibel.distanthorizons.core.enums.EDhDirection;
+import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
 import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider;
 import com.seibel.distanthorizons.core.level.IDhClientLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -36,6 +37,7 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
 import org.apache.logging.log4j.Logger;
 
 import java.awt.*;
+import java.util.ArrayList;
 import java.util.Objects;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.CompletableFuture;
@@ -84,6 +86,10 @@ public class LodRenderSection implements IDebugRenderable
 	
 	private final QuadTree<LodRenderSection> parentQuadTree;
 	
+	private boolean missingPositionsCalculated = false;
+	/** should be an empty array if no positions need to be generated */
+	private ArrayList<DhSectionPos> missingGenerationPos = null;
+	
 	
 	
 	//=============//
@@ -407,6 +413,56 @@ public class LodRenderSection implements IDebugRenderable
 	
 	
 	
+	//=================================//
+	// full data retrieval (world gen) //
+	//=================================//
+	
+	public boolean isFullyGenerated() { return this.missingPositionsCalculated && this.missingGenerationPos.size() == 0; }
+	public boolean missingPositionsCalculated() { return this.missingPositionsCalculated; }
+	public int ungeneratedPositionCount() { return (this.missingGenerationPos != null) ? this.missingGenerationPos.size() : 0; }
+	
+	public void tryQueuingMissingLodRetrieval(IFullDataSourceProvider fullDataSourceProvider)
+	{
+		if (fullDataSourceProvider.canRetrieveMissingDataSources() && fullDataSourceProvider.canQueueRetrieval())
+		{
+			// calculate the missing positions if not already done
+			if (!this.missingPositionsCalculated)
+			{
+				this.missingGenerationPos = fullDataSourceProvider.getPositionsToRetrieve(this.pos);
+				if (this.missingGenerationPos != null)
+				{
+					this.missingPositionsCalculated = true;
+				}
+			}
+			
+			// if the missing positions were found, queue them
+			if (this.missingGenerationPos != null)
+			{
+				// queue from last to first to prevent shifting the array unnecessarily
+				for (int i = this.missingGenerationPos.size() - 1; i >= 0; i--)
+				{
+					if (!fullDataSourceProvider.canQueueRetrieval())
+					{
+						// the data source provider isn't accepting any more jobs
+						break;
+					}
+					
+					DhSectionPos pos = this.missingGenerationPos.remove(i);
+					boolean positionQueued = fullDataSourceProvider.queuePositionForRetrieval(pos);
+					if (!positionQueued)
+					{
+						// shouldn't normally happen, but just in case
+						this.missingGenerationPos.add(pos);
+						break;
+					}
+				}
+			}
+		}
+	}
+	
+	
+	
+	
 	//==============//
 	// base methods //
 	//==============//

core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java

@@ -209,6 +209,26 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
 		return list;
 	}
 	
+	/** @return null if nothing exists for this position */
+	public byte[] getColumnGenerationStepForPos(DhSectionPos pos)
+	{
+		int detailLevel = pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
+		
+		Map<String, Object> resultMap = this.queryDictionaryFirst(
+				"select ColumnGenerationStep " +
+						"from "+this.getTableName()+" " +
+						"WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ());
+		
+		if (resultMap != null)
+		{
+			return (byte[]) resultMap.get("ColumnGenerationStep");
+		}
+		else
+		{
+			return null;
+		}
+	}
+	
 	
 	
 }