Handle generated LOD updates outside the render thread

This reverts commit 280181c91e.

api/src/main/java/com/seibel/distanthorizons/api/interfaces/override/worldGenerator/IDhApiWorldGenerator.java

@@ -169,7 +169,7 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
 	 *
 	 * After the {@link IDhApiWorldGenerator} has been generated, it should be passed into the
 	 * resultConsumer's {@link Consumer#accept(Object)} method.
-	 * Note: if air blocks aren't included in the with the {@link DhApiChunk} with proper lighting, lower detail levels will appear as black/unlit.
+	 * Note: if air blocks aren't included in the {@link IDhApiFullDataSource} with proper lighting, lower detail levels will appear as black/unlit.
 	 *
 	 * @implNote the default implementation of this method throws an {@link UnsupportedOperationException},
 	 * and must be overridden when {@link #getReturnType()} returns {@link EDhApiWorldGeneratorReturnType#API_CHUNKS}.

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java

@@ -121,7 +121,7 @@ public class FullDataSourceV2
 	public Boolean applyToChildren = null;
 	
 	/** should only be used by methods exposed via the DH API */
-	private boolean runApiChunkValidation = false;
+	private boolean runApiSetterValidation = false;
 	
 	
 	
@@ -1296,7 +1296,7 @@ public class FullDataSourceV2
 	// API methods //
 	//=============//
 	
-	public void setRunApiChunkValidation(boolean runValidation) { this.runApiChunkValidation = runValidation; }
+	public void setRunApiSetterValidation(boolean runValidation) { this.runApiSetterValidation = runValidation; }
 	
 	@Override
 	public int getWidthInDataColumns() { return WIDTH; }
@@ -1308,7 +1308,7 @@ public class FullDataSourceV2
 		try
 		{
 			LodDataBuilder.putListInTopDownOrder(columnDataPoints);
-			if (this.runApiChunkValidation)
+			if (this.runApiSetterValidation)
 			{
 				LodDataBuilder.validateOrThrowApiDataColumn(columnDataPoints);
 			}

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java

@@ -28,8 +28,7 @@ import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataUpdatePropag
 import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
 import com.seibel.distanthorizons.core.generation.DhLightingEngine;
 import com.seibel.distanthorizons.core.generation.IFullDataSourceRetrievalQueue;
-import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
-import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
+import com.seibel.distanthorizons.core.generation.tasks.DataSourceRetrievalResult;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLogger;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -38,12 +37,12 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
+import com.seibel.distanthorizons.core.util.ExceptionUtil;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker;
 import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
 import it.unimi.dsi.fastutil.bytes.ByteArrayList;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
-import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
@@ -68,7 +67,9 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	 * TODO this should be dynamically allocated based on CPU load
 	 *  and abilities.
 	 */
-	public static final int MAX_WORLD_GEN_REQUESTS_PER_THREAD = 20; 
+	public static final int MAX_WORLD_GEN_REQUESTS_PER_THREAD = 20;
+	
+	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Generated Provider");
 	
 	
 	private final AtomicReference<IFullDataSourceRetrievalQueue> worldGenQueueRef = new AtomicReference<>(null);
@@ -85,15 +86,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure) throws SQLException, IOException 
 	{ this(level, saveStructure, null); }
 	public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) throws SQLException, IOException
-	{
-		super(level, saveStructure, saveDirOverride);
-		
-		this.addDataSourceUpdateListener((@NotNull FullDataSourceV2 updatedData) ->
-		{
-			this.onWorldGenTaskComplete(WorldGenResult.CreateSuccess(updatedData.getPos()), null);
-		});
-		
-	}
+	{ super(level, saveStructure, saveDirOverride); }
 	
 	
 	
@@ -122,30 +115,35 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	// events //
 	//========//
 	
-	private void onWorldGenTaskComplete(WorldGenResult genTaskResult, Throwable exception)
+	private void onWorldGenTaskComplete(DataSourceRetrievalResult genTaskResult, Throwable exception)
 	{
 		if (exception != null)
 		{
 			// don't log shutdown exceptions
-			if (!(exception instanceof CancellationException || exception.getCause() instanceof CancellationException))
+			if (!ExceptionUtil.isInterruptOrReject(exception))
 			{
 				LOGGER.error("Uncaught Gen Task Exception at ["+genTaskResult.pos+"], error: ["+exception.getMessage()+"].", exception);
 			}
 		}
-		else if (genTaskResult.success)
+		else if (genTaskResult.generatedDataSource != null)
 		{
+			this.dataUpdater.updateDataSource(genTaskResult.generatedDataSource);
 			this.fireOnGenPosSuccessListeners(genTaskResult.pos);
-			return;
+		}
+		else if (exception == null && !genTaskResult.success) // TODO use enum to check type
+		{
+			// task was split
 		}
 		else
 		{
-			// generation didn't complete
-			LOGGER.debug("Gen Task Failed at " + genTaskResult.pos);
+			// shouldn't happen, but just in case
+			// TODO is definitely happening
+			LOGGER.warn("Unexpected gen Task state at: [" + DhSectionPos.toString(genTaskResult.pos) + "], success: ["+genTaskResult.success+"], datasource: NULL, exception: NULL.");
 		}
 		
 		
 		// if the generation task was split up into smaller positions, add the on-complete event to them
-		for (CompletableFuture<WorldGenResult> siblingFuture : genTaskResult.childFutures)
+		for (CompletableFuture<DataSourceRetrievalResult> siblingFuture : genTaskResult.childFutures)
 		{
 			siblingFuture.whenComplete((siblingGenTaskResult, siblingEx) -> this.onWorldGenTaskComplete(siblingGenTaskResult, siblingEx));
 		}
@@ -207,10 +205,10 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	}
 	
 	@Override
-	public boolean canQueueRetrieval() { return this.canQueueRetrieval(false); }
-	public boolean canQueueRetrieval(boolean pruneWaitingTasksAboveLimit)
+	public boolean canQueueRetrievalNow() { return this.canQueueRetrievalNow(false); }
+	public boolean canQueueRetrievalNow(boolean pruneWaitingTasksAboveLimit)
 	{
-		if (!super.canQueueRetrieval())
+		if (!super.canQueueRetrievalNow())
 		{
 			return false;
 		}
@@ -272,10 +270,11 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 		int availableTaskSlots = maxWorldGenQueueCount - worldGenQueue.getWaitingTaskCount();
 		if (availableTaskSlots <= 0)
 		{
+			//if (false)
 			if (pruneWaitingTasksAboveLimit)
 			{
 				AtomicInteger tasksToCancel = new AtomicInteger(-availableTaskSlots + 1);
-				worldGenQueue.removeRetrievalRequestIf(x -> tasksToCancel.getAndDecrement() > 0);
+				worldGenQueue.removeRetrievalRequestIf(taskPos -> tasksToCancel.getAndDecrement() > 0);
 			}
 			else
 			{
@@ -288,7 +287,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	}
 	
 	@Override
-	public CompletableFuture<WorldGenResult> queuePositionForRetrieval(Long genPos)
+	public CompletableFuture<DataSourceRetrievalResult> queuePositionForRetrieval(Long genPos)
 	{
 		IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get();
 		if (worldGenQueue == null)
@@ -296,13 +295,8 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 			return null;
 		}
 		
-		WorldGenTaskTracker genTaskTracker = new WorldGenTaskTracker(genPos);
-		CompletableFuture<WorldGenResult> worldGenFuture = worldGenQueue.submitRetrievalTask(genPos, (byte) (DhSectionPos.getDetailLevel(genPos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTaskTracker);
-		worldGenFuture.whenComplete((genTaskResult, ex) ->
-		{
-			//LOGGER.info("gen task complete ["+DhSectionPos.toString(genPos)+"]");
-			//this.onWorldGenTaskComplete(genTaskResult, ex);
-		});
+		CompletableFuture<DataSourceRetrievalResult> worldGenFuture = worldGenQueue.submitRetrievalTask(genPos, (byte) (DhSectionPos.getDetailLevel(genPos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL));
+		worldGenFuture.whenComplete(this::onWorldGenTaskComplete);
 		
 		return worldGenFuture;
 	}
@@ -321,22 +315,20 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	public void clearRetrievalQueue() { this.worldGenQueueRef.set(null); }
 	
 	
-	public boolean isFullyGenerated(ByteArrayList columnGenerationSteps)
+	public boolean generationStepsAreFullyGenerated(ByteArrayList columnGenerationSteps)
 	{
 		return IntStream.range(0, columnGenerationSteps.size())
-				.noneMatch(i ->
-				{
-					byte value = columnGenerationSteps.getByte(i);
-					return value == EDhApiWorldGenerationStep.EMPTY.value
-							|| value == EDhApiWorldGenerationStep.DOWN_SAMPLED.value;
-				});
+			.noneMatch((int intValue) ->
+			{
+				byte value = columnGenerationSteps.getByte(intValue);
+				return value == EDhApiWorldGenerationStep.EMPTY.value
+					|| value == EDhApiWorldGenerationStep.DOWN_SAMPLED.value;
+			});
 	}
 	
-	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Generated Provider");
-	
 	
 	@Override
-	public LongArrayList getPositionsToRetrieve(Long pos)
+	public LongArrayList getPositionsToRetrieve(long pos)
 	{
 		IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get();
 		if (worldGenQueue == null)
@@ -352,7 +344,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 			{
 				ByteArrayList columnGenStepArray = checkout.getByteArray(0, FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH);
 				this.repo.getColumnGenerationStepForPos(pos, columnGenStepArray);
-				if (!columnGenStepArray.isEmpty())
+				if (columnGenStepArray.size() != 0)
 				{
 					boolean positionFullyGenerated = true;
 					
@@ -378,12 +370,11 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 		
 		
 		// this section is missing one or more columns, queue the missing ones for generation.
-		// TODO speed up this logic by only checking ungenerated columns
 		LongArrayList generationList = new LongArrayList();
 		
 		byte lowestGeneratorDetailLevel = (byte) Math.min(
-				worldGenQueue.lowestDataDetail()  + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL,
-				DhSectionPos.getDetailLevel(pos));
+			worldGenQueue.lowestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL,
+			DhSectionPos.getDetailLevel(pos));
 		
 		DhSectionPos.forEachChildAtDetailLevel(pos, lowestGeneratorDetailLevel, (genPos) ->
 		{
@@ -471,48 +462,13 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	// helper classes //
 	//================//
 	
-	// TODO may not be needed
-	private class WorldGenTaskTracker implements IWorldGenTaskTracker
-	{
-		/** just used when debugging/troubleshooting */
-		private final long pos;
-		
-		public WorldGenTaskTracker(long pos) { this.pos = pos; }
-		
-		
-		@Override
-		public Consumer<FullDataSourceV2> getDataSourceConsumer()
-		{
-			return (dataSource) ->
-			{
-				GeneratedFullDataSourceProvider.this.delayedFullDataSourceSaveCache.writeDataSourceToMemoryAndQueueSave(dataSource);
-			};
-		}
-		
-		@Override
-		public CompletableFuture<Boolean> shouldGenerateSplitChild(long pos)
-		{
-			return GeneratedFullDataSourceProvider.this.getAsync(pos).thenApply(fullDataSource ->
-			{
-				//noinspection TryFinallyCanBeTryWithResources
-				try
-				{
-					return !GeneratedFullDataSourceProvider.this.isFullyGenerated(fullDataSource.columnGenerationSteps);
-				}
-				finally
-				{
-					fullDataSource.close();
-				}
-			});
-		}
-		
-	}
 	private CompletableFuture<Void> onDataSourceSaveAsync(FullDataSourceV2 fullDataSource) 
 	{
 		// block lights should have been populated at the chunkWrapper stage
 		// waiting to populate the data source's skylight at this stage prevents re-lighting and
 		// allows us to reduce cross-chunk lighting issues by lighting the whole 4x4 LOD at once
-		DhLightingEngine.INSTANCE.bakeDataSourceSkyLight(fullDataSource, LodUtil.MAX_MC_LIGHT);
+		int skyLight = this.level.getLevelWrapper().hasSkyLight() ? LodUtil.MAX_MC_LIGHT : LodUtil.MIN_MC_LIGHT;
+		DhLightingEngine.INSTANCE.bakeDataSourceSkyLight(fullDataSource, skyLight);
 		
 		return this.updateDataSourceAsync(fullDataSource);
 	}
@@ -524,7 +480,6 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 	{
 		boolean shouldDoWorldGen();
 		
-		@Nullable
 		DhBlockPos2D getTargetPosForGeneration();
 		
 		/** Fired whenever a section has completed generating */

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java

@@ -26,6 +26,8 @@ import com.seibel.distanthorizons.core.generation.RemoteWorldRetrievalQueue;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.level.LodRequestModule;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.multiplayer.client.ENetRequestState;
+import com.seibel.distanthorizons.core.multiplayer.client.NetRequestResult;
 import com.seibel.distanthorizons.core.multiplayer.client.SyncOnLoadRequestQueue;
 import com.seibel.distanthorizons.core.logging.DhLogger;
 import org.jetbrains.annotations.Nullable;
@@ -74,7 +76,7 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide
 	//==================//
 	
 	@Override
-	public boolean canQueueRetrieval() { return this.canQueueRetrieval(true); }
+	public boolean canQueueRetrievalNow() { return this.canQueueRetrievalNow(true); }
 	
 	@Override
 	@Nullable
@@ -102,10 +104,23 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide
 		Long timestamp = this.getTimestampForPos(pos);
 		if (timestamp != null)
 		{
-			this.syncOnLoadRequestQueue.submitRequest(pos, timestamp, fullDataSource ->
-			{
-				this.updateDataSourceAsync(fullDataSource).whenComplete((result, throwable) -> fullDataSource.close());
-			});
+			this.syncOnLoadRequestQueue.submitRequest(pos, timestamp)
+				.thenAccept((NetRequestResult netRequestResult) ->
+				{
+					if (netRequestResult.state == ENetRequestState.SUCCESS)
+					{
+						FullDataSourceV2 fullDataSource = netRequestResult.receivedDataSource;
+						if (fullDataSource != null)
+						{
+							this.updateDataSourceAsync(fullDataSource)
+								.handle((voidObj, throwable) -> 
+								{
+									fullDataSource.close();
+									return null;
+								});
+						}
+					}
+				});
 		}
 		
 		return super.get(pos);

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java

@@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour
 import com.seibel.distanthorizons.core.enums.EDhDirection;
 import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc;
 import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
-import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
+import com.seibel.distanthorizons.core.generation.tasks.DataSourceRetrievalResult;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLogger;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -48,7 +48,6 @@ import java.sql.SQLException;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * Handles reading/writing {@link FullDataSourceV2} 
@@ -86,9 +85,9 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable
 	protected final String levelId;
 	
 	
-	private final FullDataUpdaterV2 dataUpdater;
-	private final FullDataUpdatePropagatorV2 updatePropagator;
-	private final DataMigratorV1 dataMigratorV1;
+	protected final FullDataUpdaterV2 dataUpdater;
+	protected final FullDataUpdatePropagatorV2 updatePropagator;
+	protected final DataMigratorV1 dataMigratorV1;
 	
 	
 	
@@ -360,7 +359,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable
 	 * to the beginning of your override.
 	 * Otherwise, parent retrieval limits will be ignored.
 	 */
-	public boolean canQueueRetrieval()
+	public boolean canQueueRetrievalNow()
 	{
 		// Retrieval shouldn't happen while an unknown number of
 		// legacy data sources are present.
@@ -369,15 +368,15 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable
 	}
 	
 	/** 
-	 * @return null if this provider can't generate any positions and
+	 * @return null if this provider can't generate any positions or
 	 * an empty array if all positions were generated 
 	 */
 	@Nullable
-	public LongArrayList getPositionsToRetrieve(Long pos) { return null; }
+	public LongArrayList getPositionsToRetrieve(long pos) { return null; }
 		
-	/** @return true if the position was queued, false if not */
+	/** @return null if the position couldn't be queued */
 	@Nullable
-	public CompletableFuture<WorldGenResult> queuePositionForRetrieval(Long genPos) { return null; }
+	public CompletableFuture<DataSourceRetrievalResult> queuePositionForRetrieval(Long genPos) { return null; }
 	
 	/** does nothing if the given position isn't present in the queue */
 	public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf) { }

core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java

@@ -19,13 +19,11 @@
 
 package com.seibel.distanthorizons.core.generation;
 
-import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
-import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
+import com.seibel.distanthorizons.core.generation.tasks.DataSourceRetrievalResult;
 import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.render.LodQuadTree;
 import com.seibel.distanthorizons.core.util.objects.RollingAverage;
-import org.jetbrains.annotations.Nullable;
 
 import java.io.Closeable;
 import java.util.List;
@@ -92,7 +90,7 @@ public interface IFullDataSourceRetrievalQueue extends Closeable
 	 */
 	void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf);
 	
-	CompletableFuture<WorldGenResult> submitRetrievalTask(long pos, byte requiredDataDetail, IWorldGenTaskTracker tracker);
+	CompletableFuture<DataSourceRetrievalResult> submitRetrievalTask(long pos, byte requiredDataDetail);
 	
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java

@@ -148,7 +148,7 @@ public class PregenManager
 				this.fullDataSourceProvider.getAsync(nextSectionPos)
 					.thenAccept(fullDataSource -> 
 				{
-					if (this.fullDataSourceProvider.isFullyGenerated(fullDataSource.columnGenerationSteps))
+					if (this.fullDataSourceProvider.generationStepsAreFullyGenerated(fullDataSource.columnGenerationSteps))
 					{
 						this.pendingGenerations.invalidate(fullDataSource.getPos());
 					}

core/src/main/java/com/seibel/distanthorizons/core/generation/RemoteWorldRetrievalQueue.java

@@ -1,12 +1,13 @@
 package com.seibel.distanthorizons.core.generation;
 
 import com.seibel.distanthorizons.core.config.Config;
-import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
-import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
+import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import com.seibel.distanthorizons.core.generation.tasks.DataSourceRetrievalResult;
 import com.seibel.distanthorizons.core.level.DhClientLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.multiplayer.client.AbstractFullDataNetworkRequestQueue;
 import com.seibel.distanthorizons.core.multiplayer.client.ClientNetworkState;
+import com.seibel.distanthorizons.core.multiplayer.client.NetRequestResult;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
@@ -14,10 +15,9 @@ import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.util.WorldGenUtil;
 import com.seibel.distanthorizons.core.util.objects.RollingAverage;
 import com.seibel.distanthorizons.core.logging.DhLogger;
+import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
 
 import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
 import java.util.concurrent.*;
 
 public class RemoteWorldRetrievalQueue extends AbstractFullDataNetworkRequestQueue implements IFullDataSourceRetrievalQueue, IDebugRenderable
@@ -54,46 +54,95 @@ public class RemoteWorldRetrievalQueue extends AbstractFullDataNetworkRequestQue
 	public byte highestDataDetail() { return LodUtil.BLOCK_DETAIL_LEVEL; }
 	
 	@Override
-	public CompletableFuture<WorldGenResult> submitRetrievalTask(long sectionPos, byte requiredDataDetail, IWorldGenTaskTracker tracker)
+	public CompletableFuture<DataSourceRetrievalResult> submitRetrievalTask(long sectionPos, byte requiredDataDetail)
 	{
 		long generationStartMsTime = System.currentTimeMillis();
 		
 		
-		return super.submitRequest(sectionPos, fullDataSource -> {
-					Objects.requireNonNull(tracker.getDataSourceConsumer()).accept(fullDataSource);
-					fullDataSource.close();
-				})
-				.thenApply(requestResult ->
+		CompletableFuture<DataSourceRetrievalResult> returnFuture = new CompletableFuture<>();
+		
+		Executor worldGenExecutor = ThreadPoolUtil.getWorldGenExecutor();
+		if (worldGenExecutor == null)
+		{
+			return CompletableFuture.completedFuture(DataSourceRetrievalResult.CreateFail());
+		}
+		
+		CompletableFuture<NetRequestResult> netFuture = super.submitRequest(sectionPos, /* client timestamp */null);
+		netFuture.handle((NetRequestResult netResult, Throwable throwable) ->
+		{
+			try
+			{
+				if (throwable != null)
 				{
-					long totalGenTimeInMs = System.currentTimeMillis() - generationStartMsTime;
-					
-					int chunkWidth = DhSectionPos.getChunkWidth(sectionPos);
-					int chunkCount = chunkWidth * chunkWidth;
-					double timePerChunk = (double)totalGenTimeInMs / (double)chunkCount;
-					this.rollingAverageChunkGenTimeInMs.add(timePerChunk);
-					
-					switch (requestResult)
-					{
-						case SUCCEEDED:
-							return WorldGenResult.CreateSuccess(sectionPos);
-						case FAILED:
-							return WorldGenResult.CreateFail();
-						case REQUIRES_SPLITTING:
-							List<CompletableFuture<WorldGenResult>> childFutures = new ArrayList<>(4);
-							DhSectionPos.forEachChild(sectionPos, childPos -> {
-								tracker.shouldGenerateSplitChild(childPos).thenAccept(shouldGenerate -> {
-									if (shouldGenerate)
-									{
-										childFutures.add(this.submitRetrievalTask(childPos, requiredDataDetail, tracker));
-									}
-								});
-							});
-							return WorldGenResult.CreateSplit(childFutures);
-					}
-					
-					LodUtil.assertNotReach("Unexpected and unhandled request response result: ["+requestResult+"]");
-					return WorldGenResult.CreateFail();
-				});
+					return DataSourceRetrievalResult.CreateFail();
+				}
+				
+				long totalGenTimeInMs = System.currentTimeMillis() - generationStartMsTime;
+				
+				int chunkWidth = DhSectionPos.getChunkWidth(sectionPos);
+				int chunkCount = chunkWidth * chunkWidth;
+				double timePerChunk = (double) totalGenTimeInMs / (double) chunkCount;
+				
+				switch (netResult.state)
+				{
+					case SUCCESS:
+						// only add the time on successes
+						// it won't be a perfect estimate but fails will often come back faster, skewing the time faster
+						this.rollingAverageChunkGenTimeInMs.add(timePerChunk);
+						
+						return DataSourceRetrievalResult.CreateSuccess(sectionPos, netResult.receivedDataSource);
+					case FAIL:
+						return DataSourceRetrievalResult.CreateFail();
+					case REQUIRES_SPLITTING:
+						ArrayList<CompletableFuture<DataSourceRetrievalResult>> childFutures = new ArrayList<>(4);
+						DhSectionPos.forEachChild(sectionPos, (long childPos) ->
+						{
+							boolean shouldGenerate;
+							try (FullDataSourceV2 fullDataSource = this.level.remoteDataSourceProvider.get(childPos))
+							{
+								if (fullDataSource != null)
+								{
+									shouldGenerate = !this.level.remoteDataSourceProvider.generationStepsAreFullyGenerated(fullDataSource.columnGenerationSteps);
+								}
+								else
+								{
+									shouldGenerate = true;
+								}
+							}
+							
+							if (shouldGenerate)
+							{
+								childFutures.add(this.submitRetrievalTask(childPos, requiredDataDetail));
+							}
+						});
+						return DataSourceRetrievalResult.CreateSplit(childFutures);
+				}
+				
+				LodUtil.assertNotReach("Unexpected and unhandled request response result: [" + netResult.state + "]");
+				return DataSourceRetrievalResult.CreateFail();
+			}
+			catch (Exception e)
+			{
+				LOGGER.error("Unexpected issue in submitRetrievalTask returned future, error: ["+e.getMessage()+"]", e);
+				return DataSourceRetrievalResult.CreateFail();
+			}
+		})
+		// convert the net result
+			.handleAsync((DataSourceRetrievalResult retrievalResult, Throwable throwable) ->
+		{
+			if (throwable != null)
+			{
+				returnFuture.completeExceptionally(throwable);
+			}
+			else
+			{
+				returnFuture.complete(retrievalResult);
+			}
+			
+			return null;
+		}, worldGenExecutor);
+		
+		return returnFuture;
 	}
 	
 	@Override
@@ -109,7 +158,7 @@ public class RemoteWorldRetrievalQueue extends AbstractFullDataNetworkRequestQue
 	@Override
 	protected int getRequestRateLimit() { return this.networkState.sessionConfig.getGenerationRequestRateLimit(); }
 	@Override
-	protected boolean isSectionAllowedToGenerate(long sectionPos, DhBlockPos2D targetPos)
+	protected boolean sectionInAllowedGenerationRadius(long sectionPos, DhBlockPos2D targetPos)
 	{
 		if (this.networkState.sessionConfig.getGenerationMaxChunkRadius() > 0)
 		{
@@ -127,12 +176,13 @@ public class RemoteWorldRetrievalQueue extends AbstractFullDataNetworkRequestQue
 		return DhSectionPos.getChebyshevSignedBlockDistance(sectionPos, targetPos) <= this.networkState.sessionConfig.getMaxGenerationRequestDistance() * 16;
 	}
 	@Override
-	protected boolean onBeforeRequest(long sectionPos, CompletableFuture<ERequestResult> future)
+	protected boolean onBeforeRequest(long sectionPos, CompletableFuture<NetRequestResult> future)
 	{
-		if (DhSectionPos.getDetailLevel(sectionPos) > DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL
-				&& !Config.Server.Experimental.enableNSizedGeneration.get())
+		// split up large requests if N-sized gen isn't enabled
+		if (!Config.Server.Experimental.enableNSizedGeneration.get()
+			&& DhSectionPos.getDetailLevel(sectionPos) > DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL)
 		{
-			future.complete(ERequestResult.REQUIRES_SPLITTING);
+			future.complete(NetRequestResult.CreateSplit());
 			return false;
 		}
 		

core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java

@@ -26,11 +26,8 @@ import com.seibel.distanthorizons.api.objects.data.DhApiChunk;
 import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
-import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
-import com.seibel.distanthorizons.core.generation.tasks.InProgressWorldGenTaskGroup;
-import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
-import com.seibel.distanthorizons.core.generation.tasks.WorldGenTask;
-import com.seibel.distanthorizons.core.generation.tasks.WorldGenTaskGroup;
+import com.seibel.distanthorizons.core.generation.tasks.DataSourceRetrievalResult;
+import com.seibel.distanthorizons.core.generation.tasks.DataSourceRetrievalTask;
 import com.seibel.distanthorizons.core.level.IDhServerLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
@@ -52,14 +49,12 @@ import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
 import com.seibel.distanthorizons.core.world.DhApiWorldProxy;
 import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
 import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
-import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
 import com.seibel.distanthorizons.core.logging.DhLogger;
 
 import java.awt.*;
 import java.util.*;
 import java.util.List;
 import java.util.concurrent.*;
-import java.util.function.Consumer;
 
 public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDebugRenderable
 {
@@ -71,9 +66,8 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 	private final IDhServerLevel level;
 	
 	/** contains the positions that need to be generated */
-	private final ConcurrentHashMap<Long, WorldGenTask> waitingTasks = new ConcurrentHashMap<>();
-	
-	private final ConcurrentHashMap<Long, InProgressWorldGenTaskGroup> inProgressGenTasksByLodPos = new ConcurrentHashMap<>();
+	private final ConcurrentHashMap<Long, DataSourceRetrievalTask> waitingTasks = new ConcurrentHashMap<>();
+	private final ConcurrentHashMap<Long, DataSourceRetrievalTask> inProgressGenTasksByLodPos = new ConcurrentHashMap<>();
 	
 	/** largest numerical detail level allowed */
 	public final byte lowestDataDetail;
@@ -102,9 +96,10 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 	
 	
 	
-	//==============//
-	// constructors //
-	//==============//
+	//=============//
+	// constructor //
+	//=============//
+	///region constructor
 	
 	public WorldGenerationQueue(IDhApiWorldGenerator generator, IDhServerLevel level)
 	{
@@ -118,20 +113,29 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		LOGGER.info("Created world gen queue");
 	}
 	
+	///endregion constructor
 	
 	
-	//=================//
-	// world generator //
-	// task handling   //
-	//=================//
+	
+	//===============//
+	// task handling //
+	//===============//
+	///region task handling
 	
 	@Override
-	public CompletableFuture<WorldGenResult> submitRetrievalTask(long pos, byte requiredDataDetail, IWorldGenTaskTracker tracker)
+	public CompletableFuture<DataSourceRetrievalResult> submitRetrievalTask(long pos, byte requiredDataDetail)
 	{
 		// the generator is shutting down, don't add new tasks
 		if (this.generatorClosingFuture != null)
 		{
-			return CompletableFuture.completedFuture(WorldGenResult.CreateFail());
+			return CompletableFuture.completedFuture(DataSourceRetrievalResult.CreateFail());
+		}
+		
+		// use the existing task if present
+		DataSourceRetrievalTask existingGenTask = this.waitingTasks.get(pos);
+		if (existingGenTask != null)
+		{
+			return existingGenTask.future;
 		}
 		
 		
@@ -145,13 +149,12 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 			requiredDataDetail = this.lowestDataDetail;
 		}
 		
-		// Assert that the data at least can fill in 1 single ChunkSizedFullDataAccessor
+		// the request should be at least chunk-sized
 		LodUtil.assertTrue(DhSectionPos.getDetailLevel(pos) > requiredDataDetail + LodUtil.CHUNK_DETAIL_LEVEL);
 		
-		
-		CompletableFuture<WorldGenResult> future = new CompletableFuture<>();
-		this.waitingTasks.put(pos, new WorldGenTask(pos, requiredDataDetail, tracker, future));
-		return future;
+		DataSourceRetrievalTask genTask = new DataSourceRetrievalTask(pos, requiredDataDetail);
+		this.waitingTasks.put(pos, genTask);
+		return genTask.future;
 	}
 	
 	@Override
@@ -161,11 +164,17 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		{
 			if (removeIf.accept(genPos))
 			{
-				this.waitingTasks.remove(genPos);
+				DataSourceRetrievalTask removedTask = this.waitingTasks.remove(genPos);
+				if (removedTask != null)
+				{
+					// cancel tasks so any waiting future steps can be triggered
+					removedTask.future.cancel(true);
+				}
 			}
 		});
 	}
 	
+	///endregion task handling
 	
 	
 	
@@ -248,18 +257,27 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		}
 		
 		
-		
+		// find the closest task
 		TaskDistancePair closestTaskPair = this.waitingTasks.reduceEntries(1024,
-				entry -> new TaskDistancePair(entry.getValue(), DhSectionPos.getSectionBBoxPos(entry.getValue().pos).getCenterBlockPos().toPos2D().chebyshevDist(targetPos.toPos2D())),
-				(TaskDistancePair aTaskPair, TaskDistancePair bTaskPair) -> (aTaskPair.dist < bTaskPair.dist) ? aTaskPair : bTaskPair);
+			// get the target distance for each task
+			(Map.Entry<Long, DataSourceRetrievalTask> entry) ->
+			{ 
+				DataSourceRetrievalTask task = entry.getValue();
+				int distance = DhSectionPos.getCenterBlockPos(task.pos).chebyshevDist(targetPos);
+				return new TaskDistancePair(entry.getValue(), distance);
+			},
+			// find the closest task
+			(TaskDistancePair aTaskPair, TaskDistancePair bTaskPair) ->
+			{
+				return (aTaskPair.dist < bTaskPair.dist) ? aTaskPair : bTaskPair;
+			});
 		
 		if (closestTaskPair == null)
 		{
-			// FIXME concurrency issue
+			// the waitingTasks was modified while this check was running
 			return false;
 		}
-		
-		WorldGenTask closestTask = closestTaskPair.task;
+		DataSourceRetrievalTask closestTask = closestTaskPair.task;
 		
 		// remove the task we found, we are going to start it and don't want to run it multiple times
 		this.waitingTasks.remove(closestTask.pos, closestTask);
@@ -269,27 +287,29 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		{
 			// detail level is correct for generation, start generation
 			
-			WorldGenTaskGroup closestTaskGroup = new WorldGenTaskGroup(closestTask.pos, (byte)(closestTask.pos - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL));
-			closestTaskGroup.worldGenTasks.add(closestTask);
-			
-			if (!this.inProgressGenTasksByLodPos.containsKey(closestTask.pos))
+			DataSourceRetrievalTask existingTask = this.inProgressGenTasksByLodPos.get(closestTask.pos);
+			if (existingTask == null)
 			{
 				// no task exists for this position, start one
-				InProgressWorldGenTaskGroup newTaskGroup = new InProgressWorldGenTaskGroup(closestTaskGroup);
-				this.startWorldGenTaskGroup(newTaskGroup);
+				this.startWorldGenTaskGroup(closestTask);
 			}
 			else
 			{
-				// TODO replace the previous inProgress task if one exists
-				// Note: Due to concurrency reasons, even if the currently running task is compatible with 
-				// 		   the newly selected task, we cannot use it,
-				//         as some chunks may have already been written into.
+				// shouldn't normally happen, but if
+				// we somehow queued the same task twice:
+				// merge the two futures so they both complete
 				
-				//LOGGER.warn("A task already exists for this position, todo: "+DhSectionPos.toString(closestTask.pos));
+				existingTask.future.thenApply((DataSourceRetrievalResult result)->
+				{
+					closestTask.future.complete(result);
+					return closestTask.future; // return value ignored
+				});
+				existingTask.future.exceptionally((Throwable throwable)->
+				{
+					closestTask.future.completeExceptionally(throwable);
+					return null; // return value ignored
+				});
 			}
-			
-			// a task has been started
-			return true;
 		}
 		else
 		{
@@ -297,48 +317,50 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 			// split up the task
 			
 			
-			// split up the task and add each one to the tree
-			LinkedList<CompletableFuture<WorldGenResult>> childFutures = new LinkedList<>();
-			long sectionPos = closestTask.pos;
-			WorldGenTask finalClosestTask = closestTask;
-			DhSectionPos.forEachChild(sectionPos, (childDhSectionPos) ->
+			// split up the task and add each to the queue
+			ArrayList<CompletableFuture<DataSourceRetrievalResult>> childFutures = new ArrayList<>(4);
+			DhSectionPos.forEachChild(closestTask.pos, (childDhSectionPos) ->
 			{
-				CompletableFuture<WorldGenResult> newFuture = new CompletableFuture<>();
-				childFutures.add(newFuture);
-				
-				WorldGenTask newGenTask = new WorldGenTask(childDhSectionPos, DhSectionPos.getDetailLevel(childDhSectionPos), finalClosestTask.taskTracker, newFuture);
+				DataSourceRetrievalTask newGenTask = new DataSourceRetrievalTask(childDhSectionPos, DhSectionPos.getDetailLevel(childDhSectionPos));
+				childFutures.add(newGenTask.future);
 				this.waitingTasks.put(newGenTask.pos, newGenTask);
 			});
 			
 			// send the child futures to the future recipient, to notify them of the new tasks
-			closestTask.future.complete(WorldGenResult.CreateSplit(childFutures));
-			
-			// return true so we attempt to generate again
-			return true;
+			closestTask.future.complete(DataSourceRetrievalResult.CreateSplit(childFutures));
 		}
-	}
-	private void startWorldGenTaskGroup(InProgressWorldGenTaskGroup newTaskGroup)
-	{
-		byte taskDetailLevel = newTaskGroup.group.dataDetail;
-		long taskPos = newTaskGroup.group.pos;
-		LodUtil.assertTrue(taskDetailLevel >= this.highestDataDetail && taskDetailLevel <= this.lowestDataDetail);
 		
-		int generationRequestChunkWidthCount = BitShiftUtil.powerOfTwo(DhSectionPos.getDetailLevel(taskPos) - taskDetailLevel - 4); // minus 4 is equal to dividing by 16 to convert to chunk scale
+		
+		// a task has been started or queued
+		// queue another task
+		return true;
+	}
+	private boolean canGenerateDetailLevel(byte taskDetailLevel)
+	{
+		byte requestedDetailLevel = (byte) (taskDetailLevel - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
+		return (this.highestDataDetail <= requestedDetailLevel && requestedDetailLevel <= this.lowestDataDetail);
+	}
+	private void startWorldGenTaskGroup(DataSourceRetrievalTask worldGenTask)
+	{
+		long taskPos = worldGenTask.pos;
+		LodUtil.assertTrue(
+			worldGenTask.requestDetailLevel >= this.highestDataDetail 
+			&& worldGenTask.requestDetailLevel <= this.lowestDataDetail,
+			"World gen task started that isn't within the range that the generator can create.");
 		
 		long generationStartMsTime = System.currentTimeMillis();
-		CompletableFuture<Void> generationFuture = this.startGenerationEvent(taskPos, taskDetailLevel, generationRequestChunkWidthCount, newTaskGroup.group::consumeDataSource);
+		CompletableFuture<FullDataSourceV2> generationFuture = this.startGenerationEvent(worldGenTask);
+		
+		// calculate generation speed
 		generationFuture.thenRun(() -> 
 		{
 			long totalGenTimeInMs = System.currentTimeMillis() - generationStartMsTime;
-			int chunkCount = generationRequestChunkWidthCount * generationRequestChunkWidthCount;
+			int chunkCount = worldGenTask.widthInChunks * worldGenTask.widthInChunks;
 			double timePerChunk = (double)totalGenTimeInMs / (double)chunkCount;
 			this.rollingAverageChunkGenTimeInMs.add(timePerChunk);
 		});
 		
-		newTaskGroup.genFuture = generationFuture;
-		LodUtil.assertTrue(newTaskGroup.genFuture != null);
-		
-		newTaskGroup.genFuture.whenComplete((voidObj, exception) ->
+		generationFuture.handle((fullDataSourceV2, exception) ->
 		{
 			try
 			{
@@ -350,157 +372,48 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 						LOGGER.error("Error generating data for pos: " + DhSectionPos.toString(taskPos), exception);
 					}
 					
-					newTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateFail()));
+					worldGenTask.future.complete(DataSourceRetrievalResult.CreateFail());
 				}
-				else
-				{
-					newTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateSuccess(taskPos)));
-				}
-				boolean worked = this.inProgressGenTasksByLodPos.remove(taskPos, newTaskGroup);
+				
+				boolean worked = this.inProgressGenTasksByLodPos.remove(taskPos, worldGenTask);
 				LodUtil.assertTrue(worked, "Unable to find in progress generator task with position ["+DhSectionPos.toString(taskPos)+"]");
+				
+				worldGenTask.future.complete(DataSourceRetrievalResult.CreateSuccess(taskPos, fullDataSourceV2));
 			}
 			catch (Exception e)
 			{
 				LOGGER.error("Unexpected error completing world gen task at pos: ["+DhSectionPos.toString(taskPos)+"].", e);
+				worldGenTask.future.completeExceptionally(e);
 			}
 			finally
 			{
 				this.tryQueueNewWorldGenRequestsAsync();
 			}
+			
+			return null;
 		});
-		
-		this.inProgressGenTasksByLodPos.put(taskPos, newTaskGroup);
 	}
-	private CompletableFuture<Void> startGenerationEvent(
-		long requestPos, 
-		byte targetDataDetail,
-		int generationRequestChunkWidthCount,
-		Consumer<FullDataSourceV2> dataSourceConsumer
-		)
+	private CompletableFuture<FullDataSourceV2> startGenerationEvent(DataSourceRetrievalTask task)
 	{
-		DhChunkPos chunkPosMin = new DhChunkPos(DhSectionPos.getSectionBBoxPos(requestPos).getCornerBlockPos());
+		this.inProgressGenTasksByLodPos.put(task.pos, task);
+		
+		DhChunkPos chunkPosMin = new DhChunkPos(new DhBlockPos2D(DhSectionPos.getMinCornerBlockX(task.pos), DhSectionPos.getMinCornerBlockZ(task.pos)));
 		
 		EDhApiDistantGeneratorMode generatorMode = Config.Common.WorldGenerator.distantGeneratorMode.get();
 		EDhApiWorldGeneratorReturnType returnType = this.generator.getReturnType();
 		switch (returnType) 
 		{
 			case VANILLA_CHUNKS: 
-			{
-				return this.generator.generateChunks(
-					chunkPosMin.getX(), chunkPosMin.getZ(),
-					generationRequestChunkWidthCount,
-					targetDataDetail,
-					generatorMode,
-					ThreadPoolUtil.getWorldGenExecutor(),
-					(Object[] generatedObjectArray) -> 
-					{
-						try
-						{
-							IChunkWrapper chunkWrapper = WRAPPER_FACTORY.createChunkWrapper(generatedObjectArray);
-							
-							// only light the chunk here if necessary,
-							// lighting before this point is preferred but for potenial legacy API uses this
-							// check should be done
-							if (!chunkWrapper.isDhBlockLightingCorrect())
-							{
-								ArrayList<IChunkWrapper> nearbyChunkList = new ArrayList<>();
-								nearbyChunkList.add(chunkWrapper);
-								byte maxSkyLight = this.level.getLevelWrapper().hasSkyLight() ? LodUtil.MAX_MC_LIGHT : LodUtil.MIN_MC_LIGHT;
-								DhLightingEngine.INSTANCE.bakeChunkBlockLighting(chunkWrapper, nearbyChunkList, maxSkyLight);
-							}
-							
-							try (FullDataSourceV2 dataSource = LodDataBuilder.createFromChunk(this.level.getLevelWrapper(), chunkWrapper))
-							{
-								LodUtil.assertTrue(dataSource != null);
-								dataSourceConsumer.accept(dataSource);
-							}
-						}
-						catch (ClassCastException e)
-						{
-							LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e);
-							Config.Common.WorldGenerator.enableDistantGeneration.set(false);
-						}
-						catch (Exception e)
-						{
-							LOGGER.error("Unexpected world generator error. Error: [" + e.getMessage() + "]. World generator disabled.", e);
-							Config.Common.WorldGenerator.enableDistantGeneration.set(false);
-						}
-					}
-				);
+			{	
+				return this.startVanillaChunkGenerationEvent(task, chunkPosMin, generatorMode);
 			}
 			case API_CHUNKS: 
 			{
-				return this.generator.generateApiChunks(
-					chunkPosMin.getX(), chunkPosMin.getZ(),
-					generationRequestChunkWidthCount,
-					targetDataDetail,
-					generatorMode,
-					ThreadPoolUtil.getWorldGenExecutor(),
-					(DhApiChunk dataPoints) ->
-					{
-						try(FullDataSourceV2 dataSource = LodDataBuilder.createFromApiChunkData(dataPoints, this.generator.runApiValidation()))
-						{
-							dataSourceConsumer.accept(dataSource);
-						}
-						catch (DataCorruptedException | IllegalArgumentException e)
-						{
-							LOGGER.error("World generator returned a corrupt chunk. Error: [" + e.getMessage() + "]. World generator disabled.", e);
-							Config.Common.WorldGenerator.enableDistantGeneration.set(false);
-						}
-						catch (ClassCastException e)
-						{
-							LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e);
-							Config.Common.WorldGenerator.enableDistantGeneration.set(false);
-						}
-					}
-				);
+				return this.startApiChunkGenerationEvent(task, chunkPosMin, generatorMode);
 			}
 			case API_DATA_SOURCES:
 			{
-				// done to reduce GC overhead
-				FullDataSourceV2 pooledDataSource = FullDataSourceV2.createEmpty(requestPos);
-				// set here so the API user doesn't have to pass in this value anywhere themselves
-				pooledDataSource.setRunApiChunkValidation(this.generator.runApiValidation());
-				
-				// only apply to children if we aren't at the bottom of the tree
-				
-				pooledDataSource.applyToChildren = DhSectionPos.getDetailLevel(pooledDataSource.getPos()) > DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL;
-				pooledDataSource.applyToParent = DhSectionPos.getDetailLevel(pooledDataSource.getPos()) < DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL + 12;
-				
-				
-				return this.generator.generateLod(
-						chunkPosMin.getX(), chunkPosMin.getZ(),
-						DhSectionPos.getX(requestPos), DhSectionPos.getZ(requestPos),
-						(byte) (DhSectionPos.getDetailLevel(requestPos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL),
-						pooledDataSource,
-						generatorMode,
-						ThreadPoolUtil.getWorldGenExecutor(),
-						(IDhApiFullDataSource apiDataSource) ->
-						{
-							try
-							{
-								FullDataSourceV2 fullDataSource = (FullDataSourceV2) apiDataSource;
-								try
-								{
-									dataSourceConsumer.accept(fullDataSource);
-								}
-								finally
-								{
-									fullDataSource.close();
-								}
-							}
-							catch (IllegalArgumentException e)
-							{
-								LOGGER.error("World generator returned a corrupt data source. Error: [" + e.getMessage() + "]. World generator disabled.", e);
-								Config.Common.WorldGenerator.enableDistantGeneration.set(false);
-							}
-							catch (ClassCastException e)
-							{
-								LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e);
-								Config.Common.WorldGenerator.enableDistantGeneration.set(false);
-							}
-						}
-				);
+				return this.startApiDataSourceGenerationEvent(task, chunkPosMin, generatorMode);
 			}
 			default: 
 			{
@@ -509,30 +422,181 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 			}
 		}
 	}
+	private CompletableFuture<FullDataSourceV2> startVanillaChunkGenerationEvent(
+		DataSourceRetrievalTask task, DhChunkPos chunkPosMin, EDhApiDistantGeneratorMode generatorMode)
+	{
+		final CompletableFuture<FullDataSourceV2> returnFuture = new CompletableFuture<>();
+		
+		ArrayList<IChunkWrapper> generatedChunks = new ArrayList<>(task.widthInChunks * task.widthInChunks);
+		
+		CompletableFuture<Void> chunkGenFuture = this.generator.generateChunks(
+			chunkPosMin.getX(), chunkPosMin.getZ(),
+			task.widthInChunks,
+			task.requestDetailLevel,
+			generatorMode,
+			ThreadPoolUtil.getWorldGenExecutor(),
+			(Object[] generatedObjectArray) ->
+			{
+				try
+				{
+					IChunkWrapper chunkWrapper = WRAPPER_FACTORY.createChunkWrapper(generatedObjectArray);
+					generatedChunks.add(chunkWrapper);
+				}
+				catch (ClassCastException e)
+				{
+					LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e);
+					Config.Common.WorldGenerator.enableDistantGeneration.set(false);
+				}
+				catch (Exception e)
+				{
+					LOGGER.error("Unexpected world generator error. Error: [" + e.getMessage() + "]. World generator disabled.", e);
+					Config.Common.WorldGenerator.enableDistantGeneration.set(false);
+				}
+			}
+		);
+		
+		chunkGenFuture.exceptionally((throwable) ->
+		{
+			returnFuture.completeExceptionally(throwable);
+			return null;
+		});
+		chunkGenFuture.thenRun(() ->
+		{
+			FullDataSourceV2 requestedDataSource = FullDataSourceV2.createEmpty(task.pos);
+			
+			// process chunks //
+			for (int i = 0; i < generatedChunks.size(); i++)
+			{
+				IChunkWrapper chunkWrapper = generatedChunks.get(i);
+				
+				// only light the chunk here if necessary,
+				// lighting before this point is preferred but for legacy API use this
+				// check should be done
+				if (!chunkWrapper.isDhBlockLightingCorrect())
+				{
+					ArrayList<IChunkWrapper> nearbyChunkList = new ArrayList<>();
+					nearbyChunkList.add(chunkWrapper);
+					byte maxSkyLight = this.level.getLevelWrapper().hasSkyLight() ? LodUtil.MAX_MC_LIGHT : LodUtil.MIN_MC_LIGHT;
+					DhLightingEngine.INSTANCE.bakeChunkBlockLighting(chunkWrapper, nearbyChunkList, maxSkyLight);
+				}
+				
+				try (FullDataSourceV2 generatedDataSource = LodDataBuilder.createFromChunk(this.level.getLevelWrapper(), chunkWrapper))
+				{
+					LodUtil.assertTrue(generatedDataSource != null);
+					requestedDataSource.updateFromDataSource(generatedDataSource);
+				}
+			}
+			
+			DhLightingEngine.INSTANCE.bakeDataSourceSkyLight(requestedDataSource, LodUtil.MAX_MC_LIGHT);
+			returnFuture.complete(requestedDataSource);
+		});
+		
+		return returnFuture;
+	}
+	private CompletableFuture<FullDataSourceV2> startApiChunkGenerationEvent(
+		DataSourceRetrievalTask task,  DhChunkPos chunkPosMin, EDhApiDistantGeneratorMode generatorMode)
+	{
+		final CompletableFuture<FullDataSourceV2> returnFuture = new CompletableFuture<>();
+		
+		ArrayList<DhApiChunk> generatedChunks = new ArrayList<>(task.widthInChunks * task.widthInChunks);
+		
+		CompletableFuture<Void> chunkGenFuture = this.generator.generateApiChunks(
+			chunkPosMin.getX(), chunkPosMin.getZ(),
+			task.widthInChunks,
+			task.requestDetailLevel,
+			generatorMode,
+			ThreadPoolUtil.getWorldGenExecutor(),
+			(DhApiChunk apiChunk) -> { generatedChunks.add(apiChunk); }
+		);
+		
+		
+		chunkGenFuture.exceptionally((throwable) ->
+		{
+			returnFuture.completeExceptionally(throwable);
+			return null;
+		});
+		chunkGenFuture.thenRun(() ->
+		{
+			FullDataSourceV2 requestedDataSource = FullDataSourceV2.createEmpty(task.pos);
+			
+			for (int i = 0; i < generatedChunks.size(); i++)
+			{
+				DhApiChunk apiChunk = generatedChunks.get(i);
+				
+				try(FullDataSourceV2 generatedDataSource = LodDataBuilder.createFromApiChunkData(apiChunk, this.generator.runApiValidation()))
+				{
+					requestedDataSource.updateFromDataSource(generatedDataSource);
+				}
+				catch (DataCorruptedException | IllegalArgumentException e)
+				{
+					LOGGER.error("World generator returned a corrupt API chunk. Error: [" + e.getMessage() + "]. World generator disabled.", e);
+					Config.Common.WorldGenerator.enableDistantGeneration.set(false);
+				}
+			}
+			
+			returnFuture.complete(requestedDataSource);
+		});
+		
+		return returnFuture;
+	}
+	private CompletableFuture<FullDataSourceV2> startApiDataSourceGenerationEvent(
+		DataSourceRetrievalTask task, DhChunkPos chunkPosMin, EDhApiDistantGeneratorMode generatorMode)
+	{
+		final CompletableFuture<FullDataSourceV2> returnFuture = new CompletableFuture<>();
+		
+		
+		// done to reduce GC overhead
+		FullDataSourceV2 pooledDataSource = FullDataSourceV2.createEmpty(task.pos);
+		// set here so the API user doesn't have to pass in this value anywhere themselves
+		pooledDataSource.setRunApiSetterValidation(this.generator.runApiValidation());
+		
+		// only apply to children if we aren't at the bottom of the tree
+		pooledDataSource.applyToChildren = DhSectionPos.getDetailLevel(pooledDataSource.getPos()) > DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL;
+		pooledDataSource.applyToParent = DhSectionPos.getDetailLevel(pooledDataSource.getPos()) < DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL + 12; // TODO what does this 12 reference?
+		
+		CompletableFuture<Void> lodGenFuture = this.generator.generateLod(
+			chunkPosMin.getX(), chunkPosMin.getZ(),
+			DhSectionPos.getX(task.pos), DhSectionPos.getZ(task.pos),
+			(byte) (DhSectionPos.getDetailLevel(task.pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL),
+			pooledDataSource,
+			generatorMode,
+			ThreadPoolUtil.getWorldGenExecutor(),
+			(IDhApiFullDataSource apiDataSource) -> { }
+		);
+		
+		
+		lodGenFuture.exceptionally((throwable) ->
+		{
+			returnFuture.completeExceptionally(throwable);
+			pooledDataSource.close();
+			return null;
+		});
+		lodGenFuture.thenRun(() ->
+		{
+			returnFuture.complete(pooledDataSource);
+		});
+		
+		return returnFuture;
+	}
 	
 	
 	
 	//===================//
 	// getters / setters //
 	//===================//
+	///region getters/setters
 	
 	@Override public int getWaitingTaskCount() { return this.waitingTasks.size(); }
 	@Override public int getInProgressTaskCount() { return this.inProgressGenTasksByLodPos.size(); }
 	
-	@Override
-	public byte lowestDataDetail() { return this.lowestDataDetail; }
-	@Override
-	public byte highestDataDetail() { return this.highestDataDetail; }
+	@Override public byte lowestDataDetail() { return this.lowestDataDetail; }
+	@Override public byte highestDataDetail() { return this.highestDataDetail; }
 	
-	@Override
-	public int getEstimatedRemainingTaskCount() { return this.estimatedRemainingTaskCount; }
-	@Override
-	public void setEstimatedRemainingTaskCount(int newEstimate) { this.estimatedRemainingTaskCount = newEstimate; }
+	@Override public int getEstimatedRemainingTaskCount() { return this.estimatedRemainingTaskCount; }
+	@Override public void setEstimatedRemainingTaskCount(int newEstimate) { this.estimatedRemainingTaskCount = newEstimate; }
 	
-	@Override
-	public int getRetrievalEstimatedRemainingChunkCount() { return this.estimatedRemainingChunkCount; }
-	@Override
-	public void setRetrievalEstimatedRemainingChunkCount(int newEstimate) { this.estimatedRemainingChunkCount = newEstimate; }
+	@Override public int getRetrievalEstimatedRemainingChunkCount() { return this.estimatedRemainingChunkCount; }
+	@Override public void setRetrievalEstimatedRemainingChunkCount(int newEstimate) { this.estimatedRemainingChunkCount = newEstimate; }
 	
 	@Override 
 	public void addDebugMenuStringsToList(List<String> messageList) { }
@@ -550,13 +614,55 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		return chunkCount;
 	}
 	
+	///endregion getters/setters
+	
+	
+	
+	//=======//
+	// debug //
+	//=======//
+	///region debug
+	
+	@Override
+	public void debugRender(DebugRenderer renderer)
+	{
+		int levelMinY = this.level.getLevelWrapper().getMinHeight();
+		int levelMaxY = this.level.getLevelWrapper().getMaxHeight();
+		
+		// show the wireframe a bit lower than world max height,
+		// since most worlds don't render all the way up to the max height
+		int levelHeightRange = (levelMaxY - levelMinY);
+		int maxY = levelMaxY - (levelHeightRange / 2);
+		
+		
+		// blue - queued
+		this.waitingTasks.keySet().forEach((Long pos) -> 
+		{ 
+			renderer.renderBox(
+				new DebugRenderer.Box(pos, levelMinY, maxY, 0.05f, Color.blue)
+			); 
+		});
+		
+		// red - in progress
+		this.inProgressGenTasksByLodPos.forEach((Long pos, DataSourceRetrievalTask task) -> 
+		{ 
+			renderer.renderBox(
+				new DebugRenderer.Box(pos, levelMinY, maxY, 0.05f, Color.red)
+			); 
+		});
+	}
+	
+	///endregion debug
+	
 	
 	
 	//==========//
 	// shutdown //
 	//==========//
+	///region shutdown
 	
-	@Override public CompletableFuture<Void> startClosingAsync(boolean cancelCurrentGeneration, boolean alsoInterruptRunning)
+	@Override
+	public CompletableFuture<Void> startClosingAsync(boolean cancelCurrentGeneration, boolean alsoInterruptRunning)
 	{
 		LOGGER.info("Closing world gen queue");
 		this.queueingThread.shutdownNow();
@@ -564,33 +670,31 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		
 		// stop and remove any in progress tasks
 		ArrayList<CompletableFuture<Void>> inProgressTasksCancelingFutures = new ArrayList<>(this.inProgressGenTasksByLodPos.size());
-		this.inProgressGenTasksByLodPos.values().forEach(runningTaskGroup ->
+		this.inProgressGenTasksByLodPos.values().forEach((DataSourceRetrievalTask genTask) ->
 		{
-			CompletableFuture<Void> genFuture = runningTaskGroup.genFuture; // Do this to prevent it getting swapped out
-			if (genFuture == null)
-			{
-				// genFuture's shouldn't be null, but sometimes they are...
-				LOGGER.info("Null gen future: "+runningTaskGroup.group.pos);
-				return;
-			}
-			
+			CompletableFuture<DataSourceRetrievalResult> genFuture = genTask.future;
 			
 			if (cancelCurrentGeneration)
 			{
 				genFuture.cancel(alsoInterruptRunning);
 			}
 			
-			inProgressTasksCancelingFutures.add(genFuture.handle((voidObj, exception) ->
+			inProgressTasksCancelingFutures.add(genFuture.handle((DataSourceRetrievalResult result, Throwable throwable) ->
 			{
-				if (exception instanceof CompletionException)
+				if (throwable instanceof CompletionException)
 				{
-					exception = exception.getCause();
+					throwable = throwable.getCause();
 				}
 				
-				if (!UncheckedInterruptedException.isInterrupt(exception) 
-					&& !(exception instanceof CancellationException))
+				if (!UncheckedInterruptedException.isInterrupt(throwable)
+					&& !(throwable instanceof CancellationException))
 				{
-					LOGGER.error("Error when terminating data generation for pos: ["+DhSectionPos.toString(runningTaskGroup.group.pos)+"], error: ["+exception.getMessage()+"].", exception);
+					LOGGER.error("Error when terminating data generation for pos: ["+DhSectionPos.toString(genTask.pos)+"], error: ["+throwable.getMessage()+"].", throwable);
+				}
+				
+				if (result.generatedDataSource != null)
+				{
+					result.generatedDataSource.close();
 				}
 				
 				return null;
@@ -623,7 +727,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 			LOGGER.info("World generator thread pool shutdown with [" + queueSize + "] incomplete tasks.");
 		}
 		
-		this.inProgressGenTasksByLodPos.values().forEach((inProgressWorldGenTaskGroup) -> inProgressWorldGenTaskGroup.genFuture.cancel(true));
+		this.inProgressGenTasksByLodPos.values().forEach((inProgressWorldGenTaskGroup) -> inProgressWorldGenTaskGroup.future.cancel(true));
 		this.waitingTasks.values().forEach((worldGenTask) -> worldGenTask.future.cancel(true));
 		
 		
@@ -644,63 +748,22 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		LOGGER.info("Finished closing " + WorldGenerationQueue.class.getSimpleName());
 	}
 	
-	
-	
-	//=======//
-	// debug //
-	//=======//
-	
-	@Override
-	public void debugRender(DebugRenderer renderer)
-	{
-		int levelMinY = this.level.getLevelWrapper().getMinHeight();
-		int levelMaxY = this.level.getLevelWrapper().getMaxHeight();
-		
-		// show the wireframe a bit lower than world max height,
-		// since most worlds don't render all the way up to the max height
-		int levelHeightRange = (levelMaxY - levelMinY);
-		int maxY = levelMaxY - (levelHeightRange / 2);
-		
-		
-		// blue - queued
-		this.waitingTasks.keySet().forEach((pos) -> 
-		{ 
-			renderer.renderBox(
-					new DebugRenderer.Box(pos, levelMinY, maxY, 0.05f, Color.blue)); 
-		});
-		
-		// red - in progress
-		this.inProgressGenTasksByLodPos.forEach((pos, t) -> 
-		{ 
-			renderer.renderBox(
-					new DebugRenderer.Box(pos, levelMinY, maxY, 0.05f, Color.red)); 
-		});
-	}
-	
-	
-	
-	//================//
-	// helper methods //
-	//================//
-	
-	private boolean canGenerateDetailLevel(byte taskDetailLevel)
-	{
-		byte requestedDetailLevel = (byte) (taskDetailLevel - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
-		return (this.highestDataDetail <= requestedDetailLevel && requestedDetailLevel <= this.lowestDataDetail);
-	}
+	///endregion shutdown
 	
 	
 	
 	//================//
 	// helper classes //
 	//================//
+	///region helper classes
 	
+	/** Used during task starting to determine the closest task */
 	private static class TaskDistancePair
 	{
-		public final WorldGenTask task;
+		public final DataSourceRetrievalTask task;
 		public final int dist;
 		
-		public TaskDistancePair(WorldGenTask task, int dist)
+		public TaskDistancePair(DataSourceRetrievalTask task, int dist)
 		{
 			this.task = task;
 			this.dist = dist;
@@ -708,4 +771,8 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
 		
 	}
 	
+	///endregion helper classes
+	
+	
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/DataSourceRetrievalResult.java

@@ -19,27 +19,41 @@
 
 package com.seibel.distanthorizons.core.generation.tasks;
 
-import java.util.Collection;
-import java.util.LinkedList;
+import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
 import java.util.concurrent.CompletableFuture;
 
-public class WorldGenResult
+/**
+ * @see DataSourceRetrievalTask
+ */
+public class DataSourceRetrievalResult
 {
 	/** true if terrain was generated */
-	public final boolean success;
+	public final boolean success; // TODO reponse enum?
 	/** the position that was generated, will be null if nothing was generated */
 	public final long pos;
+	@Nullable
+	public final FullDataSourceV2 generatedDataSource;
+	
 	/** if a position is too high detail for world generator to handle it, these futures are for its 4 children positions after being split up. */
-	public final LinkedList<CompletableFuture<WorldGenResult>> childFutures = new LinkedList<>();
+	public final ArrayList<CompletableFuture<DataSourceRetrievalResult>> childFutures = new ArrayList<>(4);
 	
 	
-	public static WorldGenResult CreateSplit(Collection<CompletableFuture<WorldGenResult>> siblingFutures) { return new WorldGenResult(false, 0, siblingFutures); }
-	public static WorldGenResult CreateFail() { return new WorldGenResult(false, 0, null); }
-	public static WorldGenResult CreateSuccess(long pos) { return new WorldGenResult(true, pos, null); }
-	private WorldGenResult(boolean success, long pos, Collection<CompletableFuture<WorldGenResult>> childFutures)
+	
+	//==============//
+ 	// constructors //
+	//==============//
+	
+	public static DataSourceRetrievalResult CreateSplit(ArrayList<CompletableFuture<DataSourceRetrievalResult>> siblingFutures) { return new DataSourceRetrievalResult(false, 0, null, siblingFutures); }
+	public static DataSourceRetrievalResult CreateFail() { return new DataSourceRetrievalResult(false, 0, null,null); }
+	public static DataSourceRetrievalResult CreateSuccess(long pos, FullDataSourceV2 generatedDataSource) { return new DataSourceRetrievalResult(true, pos, generatedDataSource, null); }
+	private DataSourceRetrievalResult(boolean success, long pos, @Nullable FullDataSourceV2 generatedDataSource, ArrayList<CompletableFuture<DataSourceRetrievalResult>> childFutures)
 	{
 		this.success = success;
 		this.pos = pos;
+		this.generatedDataSource = generatedDataSource;
 		
 		if (childFutures != null)
 		{

core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/DataSourceRetrievalTask.java

@@ -19,29 +19,37 @@
 
 package com.seibel.distanthorizons.core.generation.tasks;
 
-import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
 
 import java.util.concurrent.CompletableFuture;
 
 /**
- * @author Leetom
- * @version 2022-11-25
+ * @see DataSourceRetrievalResult
  */
-public final class WorldGenTask
+public final class DataSourceRetrievalTask
 {
 	public final long pos;
-	public final byte dataDetailLevel;
-	public final IWorldGenTaskTracker taskTracker;
-	public final CompletableFuture<WorldGenResult> future;
+	/** 
+	 * Usually the same as {@link DataSourceRetrievalTask#pos}, but
+	 * can differ if the task needs something different.
+	 */
+	public final byte requestDetailLevel;
+	public final int widthInChunks;
+	
+	public final CompletableFuture<DataSourceRetrievalResult> future = new CompletableFuture<>();
 	
 	
 	
-	public WorldGenTask(long pos, byte dataDetail, IWorldGenTaskTracker taskTracker, CompletableFuture<WorldGenResult> future)
+	//=============//
+ 	// constructor //
+	//=============//
+	
+	public DataSourceRetrievalTask(long pos, byte dataDetail)
 	{
-		this.dataDetailLevel = dataDetail;
 		this.pos = pos;
-		this.taskTracker = taskTracker;
-		this.future = future;
+		this.requestDetailLevel = dataDetail;
+		this.widthInChunks = BitShiftUtil.powerOfTwo(DhSectionPos.getDetailLevel(this.pos) - this.requestDetailLevel - 4); // minus 4 is equal to dividing by 16 to convert to chunk scale 
 	}
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/InProgressWorldGenTaskGroup.java

@@ -1,39 +0,0 @@
-/*
- *    This file is part of the Distant Horizons mod
- *    licensed under the GNU LGPL v3 License.
- *
- *    Copyright (C) 2020 James Seibel
- *
- *    This program is free software: you can redistribute it and/or modify
- *    it under the terms of the GNU Lesser General Public License as published by
- *    the Free Software Foundation, version 3.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *    GNU Lesser General Public License for more details.
- *
- *    You should have received a copy of the GNU Lesser General Public License
- *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.seibel.distanthorizons.core.generation.tasks;
-
-import java.util.concurrent.CompletableFuture;
-
-/**
- * @author Leetom
- * @version 2022-11-25
- */
-public final class InProgressWorldGenTaskGroup
-{
-	public final WorldGenTaskGroup group;
-	public CompletableFuture<Void> genFuture = null;
-	
-	
-	public InProgressWorldGenTaskGroup(WorldGenTaskGroup group)
-	{
-		this.group = group;
-	}
-	
-}

core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java

@@ -1,67 +0,0 @@
-/*
- *    This file is part of the Distant Horizons mod
- *    licensed under the GNU LGPL v3 License.
- *
- *    Copyright (C) 2020 James Seibel
- *
- *    This program is free software: you can redistribute it and/or modify
- *    it under the terms of the GNU Lesser General Public License as published by
- *    the Free Software Foundation, version 3.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *    GNU Lesser General Public License for more details.
- *
- *    You should have received a copy of the GNU Lesser General Public License
- *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.seibel.distanthorizons.core.generation.tasks;
-
-import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.function.Consumer;
-
-/**
- * @author Leetom
- * @version 2022-11-25
- */
-@Deprecated // TODO look into how these are used and if they should continue to be used
-public final class WorldGenTaskGroup
-{
-	public final long pos;
-	public byte dataDetail;
-	/** Only accessed by the generator polling thread */
-	public final LinkedList<WorldGenTask> worldGenTasks = new LinkedList<>();
-	
-	
-	
-	public WorldGenTaskGroup(long pos, byte dataDetail)
-	{
-		this.pos = pos;
-		this.dataDetail = dataDetail;
-	}
-	
-	public void consumeDataSource(FullDataSourceV2 dataSource)
-	{
-		Iterator<WorldGenTask> tasks = this.worldGenTasks.iterator();
-		while (tasks.hasNext())
-		{
-			WorldGenTask task = tasks.next();
-			Consumer<FullDataSourceV2> dataSourceConsumer = task.taskTracker.getDataSourceConsumer();
-			if (dataSourceConsumer == null)
-			{
-				tasks.remove();
-				task.future.complete(WorldGenResult.CreateFail());
-			}
-			else
-			{
-				dataSourceConsumer.accept(dataSource);
-			}
-		}
-	}
-	
-}

core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhServerLevel.java

@@ -102,13 +102,12 @@ public abstract class AbstractDhServerLevel extends AbstractDhLevel implements I
 	{ return Config.Common.WorldGenerator.enableDistantGeneration.get() && !this.worldGenPlayerCenteringQueue.isEmpty(); }
 	
 	@Override
-	@Nullable
 	public DhBlockPos2D getTargetPosForGeneration()
 	{
 		IServerPlayerWrapper firstPlayer = this.worldGenPlayerCenteringQueue.peek();
 		if (firstPlayer == null)
 		{
-			return null;
+			return DhBlockPos2D.ZERO;
 		}
 		
 		// Put first player in back before removing from front, so it can be removed by other thread without blocking

core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java

@@ -247,7 +247,6 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
 	}
 	
 	@Override
-	@Nullable
 	public DhBlockPos2D getTargetPosForGeneration() { return new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos()); }
 	
 	
@@ -259,13 +258,6 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
 	@Override
 	public void onWorldGenTaskComplete(long pos)
 	{
-		DebugRenderer.makeParticle(
-				new DebugRenderer.BoxParticle(
-						new DebugRenderer.Box(pos, 128f, 156f, 0.09f, Color.red.darker()),
-						0.2, 32f
-				)
-		);
-		
 		this.clientside.reloadPos(pos);
 	}
 	
@@ -379,9 +371,9 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
 	
 	private static class LodRequestState extends LodRequestModule.AbstractLodRequestState
 	{
-		LodRequestState(DhClientLevel level, ClientNetworkState networkState)
+		LodRequestState(DhClientLevel clientLevel, ClientNetworkState networkState)
 		{
-			this.retrievalQueue = new RemoteWorldRetrievalQueue(networkState, level);
+			this.retrievalQueue = new RemoteWorldRetrievalQueue(networkState, clientLevel);
 		}
 	}
 	

core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java

@@ -132,14 +132,6 @@ public class DhClientServerLevel extends AbstractDhServerLevel implements IDhCli
 	public void onWorldGenTaskComplete(long pos)
 	{
 		super.onWorldGenTaskComplete(pos);
-		
-		DebugRenderer.makeParticle(
-				new DebugRenderer.BoxParticle(
-						new DebugRenderer.Box(pos, 128f, 156f, 0.09f, Color.red.darker()),
-						0.2, 32f
-				)
-		);
-		
 		this.clientside.reloadPos(pos);
 	}
 	

core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java

@@ -56,7 +56,7 @@ public class DhServerLevel extends AbstractDhServerLevel
 		return true; //todo;
 	}
 	@Override
-	public @Nullable DhBlockPos2D getTargetPosForGeneration()
+	public DhBlockPos2D getTargetPosForGeneration()
 	{
 		DhBlockPos2D targetPos = super.getTargetPosForGeneration();
 		if (targetPos == null)

core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java

@@ -23,7 +23,6 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
 import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.util.ratelimiting.SupplierBasedRateLimiter;
-import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
 import com.seibel.distanthorizons.core.world.DhApiWorldProxy;
 import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
 
@@ -33,9 +32,7 @@ import java.awt.*;
 import java.util.*;
 import java.util.List;
 import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Consumer;
 
 public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRenderable, AutoCloseable
 {
@@ -58,7 +55,7 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 	
 	private volatile CompletableFuture<Void> closingFuture = null;
 	
-	protected final ConcurrentMap<Long, RequestQueueEntry> waitingTasksBySectionPos = new ConcurrentHashMap<>();
+	protected final ConcurrentMap<Long, NetRequestTask> waitingTasksBySectionPos = new ConcurrentHashMap<>();
 	/**
 	 * This semaphore prevents a given thread from accidentally locking on the same group
 	 * multiple times, as the semaphore is tied to the given thread. <br>
@@ -108,8 +105,8 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 	//==================//
 	
 	protected abstract int getRequestRateLimit();
-	protected abstract boolean isSectionAllowedToGenerate(long sectionPos, DhBlockPos2D targetPos);
-	protected abstract boolean onBeforeRequest(long sectionPos, CompletableFuture<ERequestResult> future);
+	protected abstract boolean sectionInAllowedGenerationRadius(long sectionPos, DhBlockPos2D targetPos);
+	protected abstract boolean onBeforeRequest(long sectionPos, CompletableFuture<NetRequestResult> future);
 	
 	protected abstract String getQueueName();
 	
@@ -119,45 +116,44 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 	// request submitting //
 	//====================//
 	
-	public CompletableFuture<ERequestResult> submitRequest(long sectionPos, Consumer<FullDataSourceV2> dataSourceConsumer)
-	{ return this.submitRequest(sectionPos, null, dataSourceConsumer); }
-	public CompletableFuture<ERequestResult> submitRequest(long sectionPos, @Nullable Long clientTimestamp, Consumer<FullDataSourceV2> dataSourceConsumer)
+	public CompletableFuture<NetRequestResult> submitRequest(long sectionPos, @Nullable Long clientTimestamp)
 	{
 		if (this.succeededPositions.contains(sectionPos))
 		{
-			return CompletableFuture.completedFuture(ERequestResult.FAILED);
+			return CompletableFuture.completedFuture(NetRequestResult.CreateFail());
 		}
 		
 		if (this.requiresSplittingPositions.contains(sectionPos))
 		{
-			return CompletableFuture.completedFuture(ERequestResult.REQUIRES_SPLITTING);
+			return CompletableFuture.completedFuture(NetRequestResult.CreateSplit());
 		}
 		
-		AtomicBoolean added = new AtomicBoolean(false);
-		RequestQueueEntry entry = this.waitingTasksBySectionPos.compute(sectionPos, (pos, existingQueueEntry) ->
+		NetRequestTask requestEntry = this.waitingTasksBySectionPos.compute(sectionPos, (pos, existingNetTask) ->
 		{
-			if (existingQueueEntry != null)
+			// ignore already queued tasks
+			if (existingNetTask != null)
 			{
-				return existingQueueEntry;
+				return existingNetTask;
 			}
 			
-			RequestQueueEntry newEntry = new RequestQueueEntry(dataSourceConsumer, clientTimestamp);
-			newEntry.future.whenComplete((requestResult, throwable) ->
+			
+			NetRequestTask newRequestEntry = new NetRequestTask(pos, clientTimestamp);
+			newRequestEntry.future.whenComplete((requestResult, throwable) ->
 			{
-				this.waitingTasksBySectionPos.remove(sectionPos);
+				this.waitingTasksBySectionPos.remove(pos);
 				
-				switch (requestResult)
+				switch (requestResult.state)
 				{
-					case SUCCEEDED:
+					case SUCCESS:
 						this.finishedRequests.incrementAndGet();
 						this.succeededPositions.add(pos);
-						return;
+						break;
 					case REQUIRES_SPLITTING:
-						this.requiresSplittingPositions.add(sectionPos);
-						return;
-					case FAILED:
+						this.requiresSplittingPositions.add(pos);
+						break;
+					case FAIL:
 						this.failedRequests.incrementAndGet();
-						return;
+						break;
 					default:
 						if (throwable != null && !(throwable instanceof CancellationException))
 						{
@@ -167,26 +163,22 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 				}
 			});
 			
-			added.set(true);
-			return newEntry;
+			return newRequestEntry;
 		});
 		
-		if (!added.get())
-		{
-			return CompletableFuture.completedFuture(ERequestResult.FAILED);
-		}
-		
-		return entry.future;
+		return requestEntry.future;
 	}
 	
 	public synchronized boolean tick(DhBlockPos2D targetPos)
 	{
-		if (DhApiWorldProxy.INSTANCE.worldLoaded() && DhApiWorldProxy.INSTANCE.getReadOnly())
+		if (DhApiWorldProxy.INSTANCE.worldLoaded() 
+			&& DhApiWorldProxy.INSTANCE.getReadOnly())
 		{
 			return false;
 		}
 		
-		if (this.closingFuture != null || !this.networkState.isReady())
+		if (this.closingFuture != null 
+			|| !this.networkState.isReady())
 		{
 			return false;
 		}
@@ -209,145 +201,125 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 	}
 	private void sendNextRequest(DhBlockPos2D targetPos)
 	{
-		Map.Entry<Long, RequestQueueEntry> mapEntry = this.waitingTasksBySectionPos.entrySet().stream()
-				.filter(task -> task.getValue().networkDataSourceFuture == null)
-				.min(Comparator.comparingInt(x -> DhSectionPos.getChebyshevSignedBlockDistance(x.getKey(), targetPos)))
-				.orElse(null);
+		Map.Entry<Long, NetRequestTask> nearestMapEntry = this.waitingTasksBySectionPos
+			.entrySet().stream()
+			.filter(task -> task.getValue().networkDataSourceFuture == null)
+			.min(Comparator.comparingInt(mapEntry -> DhSectionPos.getChebyshevSignedBlockDistance(mapEntry.getKey(), targetPos)))
+			.orElse(null);
 		
-		if (mapEntry == null)
+		if (nearestMapEntry == null)
 		{
 			this.pendingTasksSemaphore.release();
 			return;
 		}
 		
-		long sectionPos = mapEntry.getKey();
-		RequestQueueEntry entry = mapEntry.getValue();
+		long requestPos = nearestMapEntry.getKey();
+		NetRequestTask requestTask = nearestMapEntry.getValue();
 		
-		if (!this.isSectionAllowedToGenerate(sectionPos, targetPos))
+		if (!this.sectionInAllowedGenerationRadius(requestPos, targetPos))
 		{
-			entry.future.cancel(false);
+			requestTask.future.cancel(false);
 			this.pendingTasksSemaphore.release();
 			return;
 		}
 		
-		if (!this.onBeforeRequest(sectionPos, entry.future))
+		if (!this.onBeforeRequest(requestPos, requestTask.future))
 		{
 			this.pendingTasksSemaphore.release();
 			return;
 		}
 		
-		Long offsetEntryTimestamp = entry.updateTimestamp != null
-				? entry.updateTimestamp + this.networkState.getServerTimeOffset()
+		Long offsetEntryTimestamp = requestTask.updateTimestamp != null
+				? requestTask.updateTimestamp + this.networkState.getServerTimeOffset()
 				: null;
 		
-		CompletableFuture<FullDataSourceResponseMessage> dataSourceFuture = this.networkState.getSession().sendRequest(
-				new FullDataSourceRequestMessage(this.level.getLevelWrapper(), sectionPos, offsetEntryTimestamp),
+		CompletableFuture<FullDataSourceResponseMessage> dataSourceNetworkFuture = this.networkState.getSession().sendRequest(
+				new FullDataSourceRequestMessage(this.level.getLevelWrapper(), requestPos, offsetEntryTimestamp),
 				FullDataSourceResponseMessage.class
 		);
-		entry.networkDataSourceFuture = dataSourceFuture;
-		dataSourceFuture.handle((response, throwable) ->
+		requestTask.networkDataSourceFuture = dataSourceNetworkFuture;
+		dataSourceNetworkFuture.handle((FullDataSourceResponseMessage response, Throwable throwable) ->
 		{
-			this.pendingTasksSemaphore.release();
-			
-			try
-			{
-				if (throwable != null)
-				{
-					throw throwable;
-				}
-				
-				if (response.payload != null)
-				{
-					FullDataSourceV2DTO dataSourceDto = this.networkState.fullDataPayloadReceiver.decodeDataSource(response.payload);
-					
-					// set application flags based on the received detail level,
-					// this is needed so the data sources propagate correctly
-					dataSourceDto.applyToChildren = DhSectionPos.getDetailLevel(dataSourceDto.pos) > DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL;
-					dataSourceDto.applyToParent = DhSectionPos.getDetailLevel(dataSourceDto.pos) < DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL + 12;
-					
-					AbstractExecutorService executor = ThreadPoolUtil.getNetworkCompressionExecutor();
-					if (executor == null)
-					{
-						LOGGER.warn("Unable to handle FullDataPayload - getNetworkCompressionExecutor() is null");
-						dataSourceDto.close();
-						return null;
-					}
-					
-					CompletableFuture.runAsync(() ->
-					{
-						try
-						{
-							this.level.updateBeaconBeamsForSectionPos(dataSourceDto.pos, response.payload.beaconBeams);
-							
-							FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper(), null);
-							entry.dataSourceConsumer.accept(fullDataSource);
-						}
-						catch (Exception e)
-						{
-							throw new RuntimeException(e);
-						}
-						finally
-						{
-							dataSourceDto.close();
-						}
-					}, executor);
-				}
-				else
-				{
-					LodUtil.assertTrue(this.changedOnly, "Received empty data source response for not changes-only request");
-				}
-			}
-			catch (SectionRequiresSplittingException ignored)
-			{
-				return entry.future.complete(ERequestResult.REQUIRES_SPLITTING);
-			}
-			catch (SessionClosedException | CancellationException ignored)
-			{
-				return entry.future.cancel(false);
-			}
-			catch (RequestRejectedException e)
-			{
-				LOGGER.info("Request rejected by the server: " + e.getMessage());
-				return entry.future.complete(ERequestResult.FAILED);
-			}
-			catch (RateLimitedException e)
-			{
-				LOGGER.info("Rate limited by server, re-queueing task [" + DhSectionPos.toString(sectionPos) + "]: " + e.getMessage());
-				
-				// Skip all requests for 1 second
-				this.rateLimiter.acquireAll();
-				
-				entry.networkDataSourceFuture = null;
-				return null;
-			}
-			catch (RequestOutOfRangeException e)
-			{
-				LOGGER.debug("Out of range, re-queueing task [" + DhSectionPos.toString(sectionPos) + "]: " + e.getMessage());
-				
-				entry.networkDataSourceFuture = null;
-				return null;
-			}
-			catch (Throwable e)
-			{
-				entry.retryAttempts--;
-				LOGGER.error("Error while fetching full data source, attempts left: {} / {}", entry.retryAttempts, MAX_RETRY_ATTEMPTS, e);
-				
-				// Retry logic
-				if (entry.retryAttempts > 0)
-				{
-					entry.networkDataSourceFuture = null;
-					return null;
-				}
-				else
-				{
-					return entry.future.complete(ERequestResult.FAILED);
-				}
-			}
-			
-			return entry.future.complete(ERequestResult.SUCCEEDED);
+			this.handleNetResponse(requestTask, response, throwable);
+			return null;
 		});
 	}
-	
+	private void handleNetResponse(NetRequestTask requestTask, FullDataSourceResponseMessage response, Throwable throwable)
+	{
+		this.pendingTasksSemaphore.release();
+		
+		try
+		{
+			if (throwable != null)
+			{
+				throw throwable;
+			}
+			
+			if (response.payload == null)
+			{
+				LodUtil.assertTrue(this.changedOnly, "Received empty data source response for not changes-only request");
+				return;
+			}
+			
+			
+			try(FullDataSourceV2DTO dataSourceDto = this.networkState.fullDataPayloadReceiver.decodeDataSource(response.payload))
+			{
+				// set application flags based on the received detail level,
+				// this is needed so the data sources propagate correctly
+				dataSourceDto.applyToChildren = DhSectionPos.getDetailLevel(dataSourceDto.pos) > DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL;
+				dataSourceDto.applyToParent = DhSectionPos.getDetailLevel(dataSourceDto.pos) < DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL + 12;
+				
+				
+				this.level.updateBeaconBeamsForSectionPos(dataSourceDto.pos, response.payload.beaconBeams);
+				
+				FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper(), null);
+				requestTask.future.complete(NetRequestResult.CreateSuccess(fullDataSource));
+			}
+		}
+		catch (SectionRequiresSplittingException ignored)
+		{
+			requestTask.future.complete(NetRequestResult.CreateSplit());
+		}
+		catch (SessionClosedException | CancellationException ignored)
+		{
+			requestTask.future.cancel(false);
+		}
+		catch (RequestRejectedException e)
+		{
+			LOGGER.info("Request rejected by the server, message: [" + e.getMessage() + "].");
+			requestTask.future.complete(NetRequestResult.CreateFail());
+		}
+		catch (RateLimitedException e)
+		{
+			LOGGER.info("Rate limited by server, re-queueing task [" + DhSectionPos.toString(requestTask.pos) + "], message: [" + e.getMessage() + "].");
+			
+			// Skip all requests for 1 second
+			this.rateLimiter.acquireAll();
+			
+			requestTask.networkDataSourceFuture = null;
+		}
+		catch (RequestOutOfRangeException e)
+		{
+			LOGGER.debug("Out of range, re-queueing task [" + DhSectionPos.toString(requestTask.pos) + "], message: [" + e.getMessage() + "].");
+			
+			requestTask.networkDataSourceFuture = null;
+		}
+		catch (Throwable e)
+		{
+			requestTask.retryAttempts--;
+			LOGGER.error("Unexpected error: ["+e.getMessage()+"] while fetching full data source, attempts left: ["+requestTask.retryAttempts+"] / ["+MAX_RETRY_ATTEMPTS+"]", e);
+			
+			// Retry logic
+			if (requestTask.retryAttempts > 0)
+			{
+				requestTask.networkDataSourceFuture = null;
+			}
+			else
+			{
+				requestTask.future.complete(NetRequestResult.CreateFail());
+			}
+		}
+	}
 	
 	
 	
@@ -357,22 +329,30 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 	
 	public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf)
 	{
-		for (Map.Entry<Long, RequestQueueEntry> mapEntry : (Iterable<? extends Map.Entry<Long, RequestQueueEntry>>) this.waitingTasksBySectionPos.entrySet().stream()
-				.sorted(Comparator.comparingInt((Map.Entry<Long, RequestQueueEntry> entry) -> DhSectionPos.getChebyshevSignedBlockDistance(entry.getKey(), Objects.requireNonNull(this.level.getTargetPosForGeneration()))).reversed())
-				::iterator)
+		// remove tasks furthest
+		Iterator<Map.Entry<Long, NetRequestTask>> farestTaskIterator = this.waitingTasksBySectionPos
+			.entrySet().stream()
+			.sorted(Comparator.comparingInt((Map.Entry<Long, NetRequestTask> entry) ->
+			{
+				Long pos = entry.getKey();
+				DhBlockPos2D targetPos = this.level.getTargetPosForGeneration();
+				return DhSectionPos.getChebyshevSignedBlockDistance(pos, targetPos);
+			}).reversed())
+			.iterator();
+		
+		while (farestTaskIterator.hasNext())
 		{
+			Map.Entry<Long, NetRequestTask> mapEntry = farestTaskIterator.next();
 			long pos = mapEntry.getKey();
-			RequestQueueEntry entry = mapEntry.getValue();
+			NetRequestTask entry = mapEntry.getValue();
 			
 			if (removeIf.accept(pos))
 			{
-				LOGGER.debug("Removing request  [" + mapEntry.getKey() + "]...");
-				
-				entry.future.cancel(false);
 				if (entry.networkDataSourceFuture != null)
 				{
 					entry.networkDataSourceFuture.cancel(false);
 				}
+				entry.future.cancel(false);
 			}
 		}
 	}
@@ -400,7 +380,7 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 			
 			do
 			{
-				for (RequestQueueEntry entry : this.waitingTasksBySectionPos.values())
+				for (NetRequestTask entry : this.waitingTasksBySectionPos.values())
 				{
 					entry.future.cancel(alsoInterruptRunning);
 					if (entry.networkDataSourceFuture != null && entry.networkDataSourceFuture.cancel(alsoInterruptRunning))
@@ -438,13 +418,31 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 			return;
 		}
 		
-		for (Map.Entry<Long, RequestQueueEntry> mapEntry : this.waitingTasksBySectionPos.entrySet())
+		DhBlockPos2D targetPos = this.level.getTargetPosForGeneration();
+		for (Map.Entry<Long, NetRequestTask> mapEntry : this.waitingTasksBySectionPos.entrySet())
 		{
-			renderer.renderBox(new DebugRenderer.Box(mapEntry.getKey(), -32f, 64f, 0.05f,
-					mapEntry.getValue().networkDataSourceFuture != null ? Color.red
-							: this.isSectionAllowedToGenerate(mapEntry.getKey(), Objects.requireNonNull(this.level.getTargetPosForGeneration())) ? Color.gray
-							: Color.darkGray
-			));
+			long pos = mapEntry.getKey();
+			NetRequestTask task = mapEntry.getValue();
+			
+			Color color;
+			if (task.networkDataSourceFuture != null)
+			{
+				color = Color.RED;
+			}
+			else
+			{
+				boolean taskInAllowedGenRadius = this.sectionInAllowedGenerationRadius(pos, targetPos);
+				if (taskInAllowedGenRadius)
+				{
+					color = Color.GRAY;
+				}
+				else
+				{
+					color = Color.DARK_GRAY;
+				}
+			}
+			
+			renderer.renderBox(new DebugRenderer.Box(pos, -32f, 64f, 0.05f, color));
 		}
 	}
 	
@@ -454,11 +452,12 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 	// helper classes //
 	//================//
 	
-	protected static class RequestQueueEntry
+	protected static class NetRequestTask
 	{
+		public final long pos;
+		
 		/** encapsulates the entire request, including client side queuing and the actual server request */
-		public final CompletableFuture<ERequestResult> future = new CompletableFuture<>();
-		public final Consumer<FullDataSourceV2> dataSourceConsumer;
+		public final CompletableFuture<NetRequestResult> future = new CompletableFuture<>();
 		/** will be null if we want to retrieve the LOD regardless of when it was last updated */
 		@Nullable
 		public final Long updateTimestamp;
@@ -477,23 +476,14 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
 		// constructor //
 		//=============//
 		
-		public RequestQueueEntry(
-				Consumer<FullDataSourceV2> dataSourceConsumer,
-				@Nullable Long updateTimestamp)
+		public NetRequestTask(long pos, @Nullable Long updateTimestamp)
 		{
-			this.dataSourceConsumer = dataSourceConsumer;
+			this.pos = pos;
 			this.updateTimestamp = updateTimestamp;
 		}
 		
 	}
 	
-	public enum ERequestResult
-	{
-		SUCCEEDED,
-		REQUIRES_SPLITTING,
-		FAILED,
-	}
-	
 	
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/ClientNetworkState.java

@@ -129,6 +129,7 @@ public class ClientNetworkState implements Closeable
 			{
 				this.serverSupportStatus = EServerSupportStatus.FULL;
 				
+				// TODO only log changes
 				LOGGER.info("Connection config has been changed: [" + message.config + "].");
 				this.sessionConfig = message.config;
 				this.configReceived = true;

core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/ENetRequestState.java

@@ -0,0 +1,15 @@
+package com.seibel.distanthorizons.core.multiplayer.client;
+
+/**
+ * SUCCESS <br>
+ * REQUIRES_SPLITTING <br>
+ * FAIL <br>
+ * 
+ * @see NetRequestResult
+ */
+public enum ENetRequestState
+{
+	SUCCESS,
+	REQUIRES_SPLITTING,
+	FAIL,
+}

core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/NetRequestResult.java

@@ -17,23 +17,31 @@
  *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.seibel.distanthorizons.core.generation.tasks;
+package com.seibel.distanthorizons.core.multiplayer.client;
 
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import org.jetbrains.annotations.Nullable;
 
-import javax.annotation.Nullable;
-import java.util.concurrent.CompletableFuture;
-import java.util.function.Consumer;
-
-/**
- * @author Leetom
- * @version 2022-11-25
- */
-public interface IWorldGenTaskTracker
+public class NetRequestResult
 {
+	public final ENetRequestState state;
 	@Nullable
-	Consumer<FullDataSourceV2> getDataSourceConsumer();
+	public final FullDataSourceV2 receivedDataSource;
+	
+	
+	
+	//==============//
+	// constructors //
+	//==============//
+	
+	public static NetRequestResult CreateFail() { return new NetRequestResult(ENetRequestState.FAIL, null); }
+	public static NetRequestResult CreateSuccess(FullDataSourceV2 receivedDataSource) { return new NetRequestResult(ENetRequestState.SUCCESS, receivedDataSource); }
+	public static NetRequestResult CreateSplit() { return new NetRequestResult(ENetRequestState.REQUIRES_SPLITTING, null); }
+	private NetRequestResult(ENetRequestState state, @Nullable FullDataSourceV2 receivedDataSource)
+	{
+		this.state = state;
+		this.receivedDataSource = receivedDataSource;
+	}
 	
-	CompletableFuture<Boolean> shouldGenerateSplitChild(long pos);
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/SyncOnLoadRequestQueue.java

@@ -35,12 +35,12 @@ public class SyncOnLoadRequestQueue extends AbstractFullDataNetworkRequestQueue
 	@Override
 	protected int getRequestRateLimit() { return this.networkState.sessionConfig.getSyncOnLoginRateLimit(); }
 	@Override
-	protected boolean isSectionAllowedToGenerate(long sectionPos, DhBlockPos2D targetPos)
+	protected boolean sectionInAllowedGenerationRadius(long sectionPos, DhBlockPos2D targetPos)
 	{
 		return DhSectionPos.getChebyshevSignedBlockDistance(sectionPos, targetPos) <= this.networkState.sessionConfig.getMaxSyncOnLoadDistance() * 16;
 	}
 	@Override
-	protected boolean onBeforeRequest(long sectionPos, CompletableFuture<ERequestResult> future) { return true; }
+	protected boolean onBeforeRequest(long sectionPos, CompletableFuture<NetRequestResult> future) { return true; }
 	
 	@Override
 	protected String getQueueName() { return "Sync On Login Queue"; }

core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayloadReceiver.java

@@ -55,12 +55,11 @@ public class FullDataPayloadReceiver implements AutoCloseable
 	public FullDataSourceV2DTO decodeDataSource(FullDataPayload payload)
 	{
 		CompositeByteBuf compositeByteBuffer = this.buffersById.get(payload.dtoBufferId);
-		LodUtil.assertTrue(compositeByteBuffer != null);
+		LodUtil.assertTrue(compositeByteBuffer != null, "decoded data source missing byte buffer");
 		
 		try
 		{
 			FullDataSourceV2DTO dataSourceDto = INetworkObject.decodeToInstance(FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding(), compositeByteBuffer);
-			LOGGER.debug("Buffer {} DTO: {}", payload.dtoBufferId, dataSourceDto);
 			return dataSourceDto;
 		}
 		finally

core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java

@@ -231,7 +231,7 @@ public class FullDataSourceRequestHandler implements AutoCloseable
 	{
 		this.fullDataSourceProvider().getAsync(pos).thenAccept(fullDataSource ->
 		{
-			if (this.fullDataSourceProvider().isFullyGenerated(fullDataSource.columnGenerationSteps))
+			if (this.fullDataSourceProvider().generationStepsAreFullyGenerated(fullDataSource.columnGenerationSteps))
 			{
 				requestGroup.fullDataSource = fullDataSource;
 				return;

core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java

@@ -40,7 +40,9 @@ public abstract class AbstractPhantomArrayList implements AutoCloseable
 	/** The Array counts can be 0 or greater. */
 	public AbstractPhantomArrayList(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount) 
 	{
-		if (byteArrayCount < 0 || shortArrayCount < 0 || longArrayCount < 0)
+		if (byteArrayCount < 0 
+			|| shortArrayCount < 0 
+			|| longArrayCount < 0)
 		{
 			throw new IllegalArgumentException("Can't get a negative number of pooled arrays.");
 		}

core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPos2D.java

@@ -75,6 +75,30 @@ public class DhBlockPos2D
 	public long distSquared(DhBlockPos2D other) { return this.distSquared(other.x, other.z); }
 	public long distSquared(int x, int z) { return MathUtil.pow2((long) this.x - x) + MathUtil.pow2((long) this.z - z); }
 	
+	/**
+	 * Returns the maximum distance along either the X or Z axis <br><br>
+	 *
+	 * Example chebyshev distance between X and every point around it: <br>
+	 * <code>
+	 * 2 2 2 2 2 <br>
+	 * 2 1 1 1 2 <br>
+	 * 2 1 X 1 2 <br>
+	 * 2 1 1 1 2 <br>
+	 * 2 2 2 2 2 <br>
+	 * </code>
+	 */
+	public int chebyshevDist(DhBlockPos2D other) { return Math.max(Math.abs(this.x - other.x), Math.abs(this.z - other.z)); }
+	
+	/**
+	 * Can be used to quickly determine the rough distance between two points<Br>
+	 * or determine the taxi cab (manhattan) distance between two points. <Br><Br>
+	 *
+	 * Manhattan distance is equivalent to determining the distance between two street intersections,
+	 * where you can only drive along each street, instead of directly to the other point.
+	 */
+	public int manhattanDist(DhBlockPos2D other) { return Math.abs(this.x - other.x) + Math.abs(this.z - other.z); }
+	
+	
 	
 	
 	//===========//

core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java

@@ -20,10 +20,10 @@
 package com.seibel.distanthorizons.core.render;
 
 import com.seibel.distanthorizons.core.config.Config;
-import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
 import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer;
 import com.seibel.distanthorizons.core.enums.EDhDirection;
 import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
+import com.seibel.distanthorizons.core.generation.tasks.DataSourceRetrievalResult;
 import com.seibel.distanthorizons.core.level.IDhClientLevel;
 import com.seibel.distanthorizons.core.logging.DhLogger;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -34,13 +34,15 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
 import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler;
 import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
 import com.seibel.distanthorizons.core.util.LodUtil;
-import com.seibel.distanthorizons.core.util.PerfRecorder;
 import com.seibel.distanthorizons.core.util.ThreadUtil;
+import com.seibel.distanthorizons.core.util.WorldGenUtil;
 import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode;
 import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree;
 import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
 import com.seibel.distanthorizons.coreapi.util.MathUtil;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.longs.LongIterator;
+import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
 import org.jetbrains.annotations.Nullable;
 
 import javax.annotation.WillNotClose;
@@ -49,7 +51,6 @@ import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -61,7 +62,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 {
 	private static final DhLogger LOGGER = new DhLoggerBuilder().build();
 	/** there should only ever be one {@link LodQuadTree} so having the thread static should be fine */
-	private static final ThreadPoolExecutor FULL_DATA_RETRIEVAL_QUEUE_THREAD = ThreadUtil.makeSingleThreadPool("QuadTree Full Data Retrieval Queue Populator");
+	private static final ThreadPoolExecutor FULL_DATA_RETRIEVAL_QUEUE_THREAD = ThreadUtil.makeSingleThreadPool("LodQuadTree Data Retrieval Queue");
 	
 	
 	public final int blockRenderDistanceDiameter;
@@ -73,9 +74,8 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 	 * This is a {@link ConcurrentLinkedQueue} because new sections can be added to this list via the world generator threads.
 	 */
 	private final ConcurrentLinkedQueue<Long> sectionsToReload = new ConcurrentLinkedQueue<>();
-	private final IDhClientLevel level; //FIXME: Proper hierarchy to remove this reference!
+	private final IDhClientLevel level;
 	private final ReentrantLock treeReadWriteLock = new ReentrantLock();
-	private final AtomicBoolean fullDataRetrievalQueueRunning = new AtomicBoolean(false);
 	
 	private ArrayList<LodRenderSection> debugRenderSections = new ArrayList<>();
 	private ArrayList<LodRenderSection> altDebugRenderSections = new ArrayList<>();
@@ -104,11 +104,16 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 	/** used to calculate when a detail drop will occur */
 	private double detailDropOffLogBase;
 	
+	/** the {@link DhSectionPos} that need to be retrieved/generated */
+	public final LongOpenHashSet missingGenerationPosSet = new LongOpenHashSet();
+	public final LongOpenHashSet queuedGenerationPosSet = new LongOpenHashSet();
 	
 	
-	//==============//
-	// constructors //
-	//==============//
+	
+	//=============//
+	// constructor //
+	//=============//
+	//region constructor
 	
 	public LodQuadTree(
 			IDhClientLevel level, int viewDiameterInBlocks,
@@ -128,11 +133,14 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		
 	}
 	
+	//endregion constructor
+	
 	
 	
 	//=============//
 	// tick update //
 	//=============//
+	//region tick update
 	
 	/**
 	 * This function updates the quadTree based on the playerPos and the current game configs (static and global)
@@ -143,23 +151,32 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 	{
 		if (this.level == null)
 		{
-			// the level hasn't finished loading yet
-			// TODO sometimes null pointers still happen, when logging back into a world (maybe the old level isn't null but isn't valid either?)
+			// the quad tree was created before a level reference was created
 			return;
 		}
 		
 		
 		
-		// this shouldn't be updated while the tree is being iterated through
-		this.updateDetailLevelVariables();
-		
 		// don't traverse the tree if it is being modified
 		if (this.treeReadWriteLock.tryLock())
 		{
+			// this shouldn't be updated while the tree is being iterated through
+			this.updateDetailLevelVariables();
+			
 			try
 			{
-				// recenter if necessary, removing out of bounds sections
-				this.setCenterBlockPos(playerPos, LodRenderSection::close);
+				// recenter if necessary...
+				this.setCenterBlockPos(playerPos, (renderSection) ->
+				{
+					//...removing out of bounds sections
+					if (renderSection != null)
+					{
+						this.fullDataSourceProvider.removeRetrievalRequestIf((long genPos) -> DhSectionPos.contains(renderSection.pos, genPos));
+						this.missingGenerationPosSet.remove(renderSection.pos);
+						this.queuedGenerationPosSet.remove(renderSection.pos);
+						renderSection.close();
+					}
+				});
 				
 				this.updateAllRenderSections(playerPos);
 			}
@@ -197,7 +214,6 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		
 		
 		// walk through each root node
-		HashSet<LodRenderSection> nodesNeedingRetrieval = new HashSet<>();
 		HashSet<LodRenderSection> nodesNeedingLoading = new HashSet<>();
 		LongIterator rootPosIterator = this.rootNodePosIterator();
 		while (rootPosIterator.hasNext())
@@ -210,17 +226,23 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 			}
 			
 			QuadNode<LodRenderSection> rootNode = this.getNode(rootPos);
-			this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false, nodesNeedingRetrieval, nodesNeedingLoading);
+			LodUtil.assertTrue(rootNode != null, "All root nodes should have been created by this point.");
+			this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false, nodesNeedingLoading);
 		}
 		
 		
 		// queue full data retrieval (world gen) requests if needed
-		if (nodesNeedingRetrieval.size() != 0
-			&& !this.fullDataRetrievalQueueRunning.get()
-			&& this.fullDataSourceProvider.canQueueRetrieval())
+		if (this.missingGenerationPosSet.size() != 0
+			&& this.fullDataSourceProvider.canQueueRetrievalNow())
 		{
-			this.fullDataRetrievalQueueRunning.set(true);
-			FULL_DATA_RETRIEVAL_QUEUE_THREAD.execute(() -> this.queueFullDataRetrievalTasks(playerPos, nodesNeedingRetrieval));
+			try
+			{
+				this.queueFullDataRetrievalTasks(playerPos);
+			}
+			catch (Exception e)
+			{
+				LOGGER.error("Unexpected error queuing retrieval tasks, error: [" + e.getMessage() + "].", e);
+			}
 		}
 		
 		
@@ -236,7 +258,6 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 			DhBlockPos2D playerPos, 
 			QuadNode<LodRenderSection> rootNode, QuadNode<LodRenderSection> quadNode, long sectionPos, 
 			boolean parentSectionIsRendering,
-			HashSet<LodRenderSection> nodesNeedingRetrieval,
 			HashSet<LodRenderSection> nodesNeedingLoading)
 	{
 		//=====================//
@@ -245,7 +266,8 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		//=====================//
 		
 		// create the node
-		if (quadNode == null && this.isSectionPosInBounds(sectionPos)) // the position bounds should only fail when at the edge of the user's render distance
+		if (quadNode == null 
+			&& this.isSectionPosInBounds(sectionPos)) // the position bounds should only fail when at the edge of the user's render distance
 		{
 			rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef));
 			quadNode = rootNode.getNode(sectionPos);
@@ -288,7 +310,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 			for (int i = 0; i < 4; i++)
 			{
 				QuadNode<LodRenderSection> childNode = quadNode.getChildByIndex(i);
-				boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, DhSectionPos.getChildByIndex(sectionPos, i), thisPosIsRendering || parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading);
+				boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, DhSectionPos.getChildByIndex(sectionPos, i), thisPosIsRendering || parentSectionIsRendering, nodesNeedingLoading);
 				allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded;
 			}
 			
@@ -347,7 +369,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 				for (int i = 0; i < 4; i++)
 				{
 					QuadNode<LodRenderSection> childNode = quadNode.getChildByIndex(i);
-					this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, DhSectionPos.getChildByIndex(sectionPos, i), parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading);
+					this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, DhSectionPos.getChildByIndex(sectionPos, i), parentSectionIsRendering, nodesNeedingLoading);
 				}
 				
 				// disabling rendering must be done after the children are enabled
@@ -368,23 +390,11 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 			
 			// prepare this section for rendering
 			if (!renderSection.gpuUploadInProgress()
-				&& renderSection.bufferContainer == null
-				// TODO this is commented out since some users reported LODs refusing to
-				//  load at their expected higher-detail levels
-				// this check is specifically for N-sized world generators where the higher quality
-				// data source may not exist yet, this is done to prevent holes while waiting for said generator
-				//&& renderSection.getFullDataSourceExists()
-				)
+				&& renderSection.bufferContainer == null)
 			{
 				nodesNeedingLoading.add(renderSection);
 			}
 			
-			// queue world gen if needed
-			if (!renderSection.isFullyGenerated())
-			{
-				nodesNeedingRetrieval.add(renderSection);
-			}
-			
 			// update debug if needed
 			if (Config.Client.Advanced.Debugging.DebugWireframe.showQuadTreeRenderStatus.get())
 			{
@@ -394,7 +404,8 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 			
 			
 			// wait for the parent to disable before enabling this section, so we don't have a hole
-			if (!parentSectionIsRendering && renderSection.canRender())
+			if (!parentSectionIsRendering 
+				&& renderSection.canRender())
 			{
 				// if rendering is already enabled we don't have to re-enable it
 				if (!renderSection.getRenderingEnabled())
@@ -425,6 +436,21 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 					// needs to be fired after the children are disabled so beacons render correctly
 					renderSection.onRenderingEnabled();
 					
+					// since this section wants to render
+					// check if it needs any generation to do so
+					LongArrayList missingPosList = this.fullDataSourceProvider.getPositionsToRetrieve(renderSection.pos);
+					if (missingPosList != null)
+					{
+						for (int i = 0; i < missingPosList.size(); i++)
+						{
+							long missingPos = missingPosList.getLong(i);
+							if (!this.queuedGenerationPosSet.contains(missingPos))
+							{
+								this.missingGenerationPosSet.add(missingPos);
+							}
+						}
+					}
+					
 				}
 			}
 			
@@ -453,9 +479,6 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 				LodRenderSection renderSection = this.getValue(pos);
 				if (renderSection != null)
 				{
-					// this data source may now exist
-					renderSection.updateFullDataSourceExists();
-					
 					if (renderSection.canRender())
 					{
 						if (renderSection.gpuUploadInProgress()
@@ -488,18 +511,22 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		for (int i = 0; i < loadSectionList.size(); i++)
 		{
 			LodRenderSection renderSection = loadSectionList.get(i);
-			if (!renderSection.gpuUploadInProgress() && renderSection.bufferContainer == null)
+			if (!renderSection.gpuUploadInProgress() 
+				&& renderSection.bufferContainer == null)
 			{
 				renderSection.uploadRenderDataToGpuAsync();
 			}
 		}
 	}
 	
+	//endregion tick update
+	
 	
 	
 	//====================//
 	// detail level logic //
 	//====================//
+	//region detail level logic
 	
 	/**
 	 * This method will compute the detail level based on player position and section pos
@@ -553,11 +580,14 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		this.minRootRenderDetailLevel = (byte) Math.max(minSectionDetailLevel, this.maxLeafRenderDetailLevel); // respect the user's selected max resolution if it is lower detail (IE they want 2x2 block, but minSectionDetailLevel is specifically for 1x1 block render resolution)
 	}
 	
+	//endregion detail level logic
 	
 	
-	//=============//
-	// render data //
-	//=============//
+	
+	//==========================//
+	// external render requests //
+	//==========================//
+	//region external render requests
 	
 	/**
 	 * Re-creates the color, render data.
@@ -565,34 +595,32 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 	 */
 	public void clearRenderDataCache()
 	{
-		if (this.treeReadWriteLock.tryLock()) // TODO make async, can lock render thread
+		try
 		{
-			try
+			this.treeReadWriteLock.lock();
+			LOGGER.info("Disposing render data...");
+			
+			// clear the tree
+			Iterator<QuadNode<LodRenderSection>> nodeIterator = this.nodeIterator();
+			while (nodeIterator.hasNext())
 			{
-				LOGGER.info("Disposing render data...");
-				
-				// clear the tree
-				Iterator<QuadNode<LodRenderSection>> nodeIterator = this.nodeIterator();
-				while (nodeIterator.hasNext())
+				QuadNode<LodRenderSection> quadNode = nodeIterator.next();
+				if (quadNode.value != null)
 				{
-					QuadNode<LodRenderSection> quadNode = nodeIterator.next();
-					if (quadNode.value != null)
-					{
-						quadNode.value.close();
-						quadNode.value = null;
-					}
+					quadNode.value.close();
+					quadNode.value = null;
 				}
-				
-				LOGGER.info("Render data cleared, please wait a moment for everything to reload...");
-			}
-			catch (Exception e)
-			{
-				LOGGER.error("Unexpected error when clearing LodQuadTree render cache: " + e.getMessage(), e);
-			}
-			finally
-			{
-				this.treeReadWriteLock.unlock();
 			}
+			
+			LOGGER.info("Render data cleared, please wait a moment for everything to reload...");
+		}
+		catch (Exception e)
+		{
+			LOGGER.error("Unexpected error when clearing LodQuadTree render cache: " + e.getMessage(), e);
+		}
+		finally
+		{
+			this.treeReadWriteLock.unlock();
 		}
 	}
 	
@@ -620,79 +648,115 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		}
 	}
 	
+	//endregion external render requests
+	
+	
 	
 	//=================================//
 	// full data retrieval (world gen) //
 	//=================================//
+	//region world gen
 	
-	private void queueFullDataRetrievalTasks(DhBlockPos2D playerPos, HashSet<LodRenderSection> nodesNeedingRetrieval)
+	private void queueFullDataRetrievalTasks(DhBlockPos2D playerPos)
 	{
-		try
+		// sort the nodes from nearest to farthest
+		LongArrayList sortedMissingPosList = new LongArrayList(this.missingGenerationPosSet);
+		sortedMissingPosList.sort((posA, posB) ->
 		{
-			// sort the nodes from nearest to farthest
-			ArrayList<LodRenderSection> nodeList = new ArrayList<>(nodesNeedingRetrieval);
-			nodeList.sort((a, b) ->
+			int aDist = DhSectionPos.getManhattanBlockDistance(posA, playerPos);
+			int bDist = DhSectionPos.getManhattanBlockDistance(posB, playerPos);
+			return Integer.compare(aDist, bDist);
+		});
+		
+		
+		
+		//==================================//
+		// add retrieval tasks to the queue //
+		//==================================//
+		
+		for (int i = 0; i < sortedMissingPosList.size(); i++)
+		{
+			if (!this.fullDataSourceProvider.canQueueRetrievalNow())
 			{
-				int aDist = DhSectionPos.getManhattanBlockDistance(a.pos, playerPos);
-				int bDist = DhSectionPos.getManhattanBlockDistance(b.pos, playerPos);
-				return Integer.compare(aDist, bDist);
-			});
+				break;
+			}
 			
-			// add retrieval tasks to the queue
-			for (int i = 0; i < nodeList.size(); i++)
+			long missingPos = sortedMissingPosList.getLong(i);
+			
+			// is this position within acceptable generator range?
+			boolean posInRange = WorldGenUtil.isPosInWorldGenRange(
+				missingPos,
+				Config.Common.WorldGenerator.generationCenterChunkX.get(), Config.Common.WorldGenerator.generationCenterChunkZ.get(),
+				Config.Common.WorldGenerator.generationMaxChunkRadius.get()
+			);
+			if (!posInRange)
 			{
-				LodRenderSection renderSection = nodeList.get(i);
-				if (!this.fullDataSourceProvider.canQueueRetrieval())
-				{
-					break;
-				}
+				continue;
+			}
+			
+			CompletableFuture<DataSourceRetrievalResult> genFuture = this.fullDataSourceProvider.queuePositionForRetrieval(missingPos);
+			boolean positionQueued = (genFuture != null && !genFuture.isCompletedExceptionally());
+			if (positionQueued)
+			{
+				this.queuedGenerationPosSet.add(missingPos);
+				this.missingGenerationPosSet.remove(missingPos);
 				
-				renderSection.tryQueuingMissingLodRetrieval();
-			}
-			
-			// calculate an estimate for the max number of chunks for the queue
-			int totalWorldGenChunkCount = 0;
-			int totalWorldGenTaskCount = 0;
-			for (int i = 0; i < nodeList.size(); i++)
-			{
-				LodRenderSection renderSection = nodeList.get(i);
-				if (!renderSection.missingPositionsCalculated())
+				genFuture.exceptionally((Throwable throwable) ->
 				{
-					// chunk count
-					int sectionWidthInChunks = DhSectionPos.getChunkWidth(renderSection.pos);
-					totalWorldGenChunkCount += sectionWidthInChunks * sectionWidthInChunks;
+					// gen task failed,
+					// requeue so we can try again in the future
 					
-					// task count
-					totalWorldGenTaskCount += renderSection.ungeneratedPositionCount();
-				}
-				else
+					this.queuedGenerationPosSet.remove(missingPos);
+					this.missingGenerationPosSet.add(missingPos);
+					return null;
+				});
+				genFuture.thenAccept((DataSourceRetrievalResult result) -> 
 				{
-					totalWorldGenChunkCount += renderSection.ungeneratedChunkCount();
+					// task finished
+					this.queuedGenerationPosSet.remove(missingPos);
 					
-					// 1 since we assume the position can be generated in a single go
-					// TODO this is a bad assumption, can we determine what the world gen supports and determine it from that?
-					totalWorldGenTaskCount += 1;
-				}
+					// if the task failed re-queue so we can try again
+					if (!result.success)
+					{
+						this.missingGenerationPosSet.add(missingPos);
+					}
+				});
 			}
+		}
+		
+		
+		
+		//==========================//
+		// calc task count estimate //
+		//==========================//
+		
+		// calculate an estimate for the max number of chunks for the queue
+		int totalWorldGenChunkCount = 0;
+		int totalWorldGenTaskCount = 0;
+		for (int i = 0; i < sortedMissingPosList.size(); i++)
+		{
+			long missingPos = sortedMissingPosList.getLong(i);
 			
-			this.fullDataSourceProvider.setEstimatedRemainingRetrievalChunkCount(totalWorldGenChunkCount);
-			this.fullDataSourceProvider.setTotalRetrievalPositionCount(totalWorldGenTaskCount);
-		}
-		catch (Exception e)
-		{
-			LOGGER.error("Unexpected error: "+e.getMessage(), e);
-		}
-		finally
-		{
-			this.fullDataRetrievalQueueRunning.set(false);
+			// chunk count
+			int sectionWidthInChunks = DhSectionPos.getChunkWidth(missingPos);
+			totalWorldGenChunkCount += sectionWidthInChunks * sectionWidthInChunks;
+			
+			// task count
+			totalWorldGenTaskCount++;
 		}
+		
+		this.fullDataSourceProvider.setEstimatedRemainingRetrievalChunkCount(totalWorldGenChunkCount);
+		this.fullDataSourceProvider.setTotalRetrievalPositionCount(totalWorldGenTaskCount);
 	}
 	
+	//endregion world gen
+	
 	
 	
 	//===========//
 	// debugging //
 	//===========//
+	//region debugging
 	
 	@Override
 	public void debugRender(DebugRenderer debugRenderer)
@@ -739,11 +803,14 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		}
 	}
 	
+	//endregion debugging
+	
 	
 	
 	//==============//
 	// base methods //
 	//==============//
+	//region base methods
 	
 	@Override
 	public void close()
@@ -783,6 +850,8 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 		LOGGER.info("Finished shutting down LodQuadTree");
 	}
 	
+	//endregion base methods
+	
 	
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java

@@ -19,8 +19,6 @@
 
 package com.seibel.distanthorizons.core.render;
 
-import com.google.common.base.Supplier;
-import com.google.common.base.Suppliers;
 import com.seibel.distanthorizons.core.config.Config;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
@@ -42,12 +40,10 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
 import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler;
 import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO;
 import com.seibel.distanthorizons.core.sql.repo.BeaconBeamRepo;
-import com.seibel.distanthorizons.core.util.WorldGenUtil;
 import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker;
 import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
 import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
 import org.jetbrains.annotations.Nullable;
 
 import javax.annotation.WillNotClose;
@@ -114,19 +110,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
 	 * different threads (buffer uploading is on the MC render thread) and need to be canceled separately.
 	 */
 	private CompletableFuture<LodBufferContainer> bufferUploadFuture = null;
-
-	/** 
-	 * should be an empty array if no positions need to be generated
-	 * 
-	 * @deprecated see the comment where this variable is set
-	 */
-	@Nullable
-	@Deprecated
-	private Supplier<LongArrayList> missingGenerationPosFunc;
-	private LongArrayList getMissingGenerationPos() { return this.missingGenerationPosFunc != null ? this.missingGenerationPosFunc.get() : null; }
-	
-	private boolean checkedIfFullDataSourceExists = false;
-	private boolean fullDataSourceExists = false;
 	
 	
 	
@@ -150,7 +133,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
 		
 		this.beaconRenderHandler = this.quadTree.beaconRenderHandler;
 		this.beaconBeamRepo = this.level.getBeaconBeamRepo();
-			
+		
 		DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus);
 	}
 	
@@ -453,121 +436,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
 	
 	
 	
-	//=================================//
-	// full data retrieval (world gen) //
-	//=================================//
-	//region full data retrieval
-	
-	public boolean isFullyGenerated()
-	{
-		LongArrayList missingGenerationPos = this.getMissingGenerationPos();
-		return missingGenerationPos != null && missingGenerationPos.isEmpty();
-	}
-	/** Returns true if an LOD exists, regardless of what data is in it */
-	public boolean getFullDataSourceExists() 
-	{  
-		if (!this.checkedIfFullDataSourceExists)
-		{
-			this.fullDataSourceExists = this.fullDataSourceProvider.repo.existsWithKey(this.pos);
-			this.checkedIfFullDataSourceExists = true;
-		}
-		
-		return this.fullDataSourceExists;
-	}
-	public void updateFullDataSourceExists() 
-	{
-		// we don't have any ability to remove LODs so we only
-		// need to check if an LOD was previously missing
-		if (!this.fullDataSourceExists)
-		{
-			this.checkedIfFullDataSourceExists = false;
-			this.getFullDataSourceExists();
-		}
-	}
-	
-	public boolean missingPositionsCalculated() { return this.getMissingGenerationPos() != null; }
-	public int ungeneratedPositionCount()
-	{
-		LongArrayList missingGenerationPos = this.getMissingGenerationPos();
-		return missingGenerationPos != null ? missingGenerationPos.size() : 0;
-	}
-	public int ungeneratedChunkCount()
-	{
-		LongArrayList missingGenerationPos = this.getMissingGenerationPos();
-		if (missingGenerationPos == null)
-		{
-			return 0;
-		}
-		
-		int chunkCount = 0;
-		// get the number of chunks each position contains
-		for (int i = 0; i < missingGenerationPos.size(); i++)
-		{
-			int chunkWidth = DhSectionPos.getChunkWidth(missingGenerationPos.getLong(i));
-			chunkCount += (chunkWidth * chunkWidth);
-		}
-		return chunkCount;
-	}
-	
-	public void tryQueuingMissingLodRetrieval()
-	{
-		if (this.fullDataSourceProvider.canRetrieveMissingDataSources() 
-			&& this.fullDataSourceProvider.canQueueRetrieval())
-		{
-			// calculate the missing positions if not already done
-			if (this.missingGenerationPosFunc == null)
-			{
-				// TODO memoization is needed for multiplayer, otherwise
-				//  new retrieval requests won't be submitted.
-				// TODO why is that the case? Shouldn't the missing positions be un-changing?
-				// TODO setting this value to low can cause world gen to slow down significantly
-				//  due to a race condition where the world gen thinks it is finished, but the results
-				//  haven't been saved to file yet, causing the gen to fire again
-				this.missingGenerationPosFunc = Suppliers.memoizeWithExpiration(
-						() -> this.fullDataSourceProvider.getPositionsToRetrieve(this.pos),
-						10, TimeUnit.MINUTES);
-			}
-			
-			LongArrayList missingGenerationPos = this.getMissingGenerationPos();
-			if (missingGenerationPos != null)
-			{
-				// queue from last to first to prevent shifting the array unnecessarily
-				for (int i = missingGenerationPos.size() - 1; i >= 0; i--)
-				{
-					if (!this.fullDataSourceProvider.canQueueRetrieval())
-					{
-						// the data source provider isn't accepting any more jobs
-						break;
-					}
-					
-					long pos = missingGenerationPos.removeLong(i);
-					
-					boolean posInRange = WorldGenUtil.isPosInWorldGenRange(
-						pos,
-						Config.Common.WorldGenerator.generationCenterChunkX.get(), Config.Common.WorldGenerator.generationCenterChunkZ.get(),
-						Config.Common.WorldGenerator.generationMaxChunkRadius.get()
-					);
-					if (!posInRange)
-					{
-						continue;
-					}
-					
-					
-					boolean positionQueued = (this.fullDataSourceProvider.queuePositionForRetrieval(pos) != null);
-					if (!positionQueued)
-					{
-						// shouldn't normally happen, but just in case
-						missingGenerationPos.add(pos);
-					}
-				}
-			}
-		}
-	}
-	
-	//endregion full data retrieval
-	
-	
-	
 	//=================//
 	// beacon handling //
 	//=================//
@@ -712,13 +580,14 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
 		{
 			// remove the task from our executor if present
 			// note: don't cancel the task since that prevents cleanup, we just don't want it to run
-			PriorityTaskPicker.Executor executor = ThreadPoolUtil.getRenderLoadingExecutor();
-			if (executor != null && !executor.isTerminated())
+			PriorityTaskPicker.Executor renderLoaderExecutor = ThreadPoolUtil.getRenderLoadingExecutor();
+			if (renderLoaderExecutor != null 
+				&& !renderLoaderExecutor.isTerminated())
 			{
 				Runnable runnable = this.getAndBuildRenderDataRunnable;
 				if (runnable != null)
 				{
-					executor.remove(runnable);
+					renderLoaderExecutor.remove(runnable);
 				}
 			}
 		}
@@ -729,14 +598,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
 			uploadFuture.cancel(true);
 		}
 		
-		
-		
-		// remove any active world gen requests that may be for this position
-		ThreadPoolExecutor executor = ThreadPoolUtil.getCleanupExecutor();
-		// while this should generally be a fast operation 
-		// this is run on a separate thread to prevent lag on the render thread
-		executor.execute(() -> this.fullDataSourceProvider.removeRetrievalRequestIf((genPos) -> DhSectionPos.contains(this.pos, genPos)));
-		
 	}
 	
 	//endregion base methods

core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/chunk/IChunkWrapper.java

@@ -126,8 +126,6 @@ public interface IChunkWrapper extends IBindable
 	
 	IBiomeWrapper getBiome(int relX, int relY, int relZ);
 	
-	boolean isStillValid();
-	
 	
 	
 	//========================//