Greatly improve Gen queue stability & improve loading times and performance

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java

@@ -10,8 +10,10 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
 
 import java.io.*;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /** 
  * WARNING: This is not THREAD-SAFE! 
@@ -23,13 +25,25 @@ import java.util.concurrent.ConcurrentHashMap;
  */
 public class FullDataPointIdMap
 {
+	// FIXME: Improve performance maybe?
+	private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
 	final ArrayList<Entry> entries = new ArrayList<>();
-	final ConcurrentHashMap<Entry, Integer> idMap = new ConcurrentHashMap<>(); // FIXME: Improve performance
+	final HashMap<Entry, Integer> idMap = new HashMap<>();
 	
+	private Entry getEntry(int id) {
+		lock.readLock().lock();
+		Entry entry = this.entries.get(id);
+		lock.readLock().unlock();
+		return entry;
+	}
 	
-	
-	public IBiomeWrapper getBiomeWrapper(int id) { return this.entries.get(id).biome; }
-	public IBlockStateWrapper getBlockStateWrapper(int id) { return this.entries.get(id).blockState; }
+	public IBiomeWrapper getBiomeWrapper(int id) {
+		return getEntry(id).biome;
+	}
+	public IBlockStateWrapper getBlockStateWrapper(int id) {
+		return getEntry(id).blockState;
+	}
 	
 	/** 
 	 * If an entry with the given values already exists nothing will 
@@ -38,11 +52,14 @@ public class FullDataPointIdMap
 	public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState)); }
 	private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry)
 	{
-		return this.idMap.computeIfAbsent(biomeBlockStateEntry, (entry) -> {
+		lock.writeLock().lock();
+		int result = this.idMap.computeIfAbsent(biomeBlockStateEntry, (entry) -> {
 			int id = this.entries.size();
 			this.entries.add(entry);
 			return id;
 		});
+		lock.writeLock().unlock();
+		return result;
 	}
 	
 	
@@ -52,24 +69,29 @@ public class FullDataPointIdMap
 	 */
 	public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap target)
 	{
+		target.lock.readLock().lock();
+		lock.writeLock().lock();
 		ArrayList<Entry> entriesToMerge = target.entries;
-		
 		int[] remappedEntryIds = new int[entriesToMerge.size()];
 		for (int i = 0; i < entriesToMerge.size(); i++)
 		{
 			remappedEntryIds[i] = this.addIfNotPresentAndGetId(entriesToMerge.get(i));
 		}
+		lock.writeLock().unlock();
+		target.lock.readLock().unlock();
 		return remappedEntryIds;
 	}
 	
 	/** Serializes all contained entries into the given stream, formatted in UTF */
 	public void serialize(DhDataOutputStream outputStream) throws IOException
 	{
+		lock.readLock().lock();
 		outputStream.writeInt(this.entries.size());
 		for (Entry entry : this.entries)
 		{
 			outputStream.writeUTF(entry.serialize());
 		}
+		lock.readLock().unlock();
 	}
 	
 	/** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */
@@ -89,12 +111,12 @@ public class FullDataPointIdMap
 	{
 		if (other == this)
 			return true;
-//        if (!(other instanceof IdBiomeBlockStateMap)) return false;
-//        IdBiomeBlockStateMap otherMap = (IdBiomeBlockStateMap) other;
-//        if (entries.size() != otherMap.entries.size()) return false;
-//        for (int i=0; i<entries.size(); i++) {
-//            if (!entries.get(i).equals(otherMap.entries.get(i))) return false;
-//        }
+/*        if (!(other instanceof FullDataPointIdMap)) return false;
+		FullDataPointIdMap otherMap = (FullDataPointIdMap) other;
+        if (entries.size() != otherMap.entries.size()) return false;
+        for (int i=0; i<entries.size(); i++) {
+            if (!entries.get(i).equals(otherMap.entries.get(i))) return false;
+        }*/
 		return false;
 	}
 	

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java

@@ -65,9 +65,11 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
 
 	@Override
 	public void debugRender(DebugRenderer r) {
+		if (pos.sectionDetailLevel > DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL) return;
+
 		IFullDataSource cached = cachedFullDataSource.get();
 		if (markedNeedUpdate)
-			r.renderBox(new DebugRenderer.Box(pos, 0, 512, 0.05f, Color.red));
+			r.renderBox(new DebugRenderer.Box(pos, 80f, 96f, 0.05f, Color.red));
 
 		Color c = Color.black;
 		if (cached != null) {
@@ -76,12 +78,15 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
 			} else {
 				c = Color.YELLOW;
 			}
+
 		} else if (dataSourceLoadFutureRef.get() != null) {
 			c = Color.BLUE;
 		} else if (doesFileExist) {
 			c = Color.RED;
 		}
-		//r.renderBox(new DebugRenderer.Box(pos, 0, 256, 0.05f, c));
+		boolean needUpdate = !this.writeQueueRef.get().queue.isEmpty() || markedNeedUpdate;
+		if (needUpdate) c = c.darker().darker();
+		r.renderBox(new DebugRenderer.Box(pos, 80f, 96f, 0.05f, c));
 	}
 
 	//TODO: use ConcurrentAppendSingleSwapContainer<LodDataSource> instead of below:
@@ -99,10 +104,12 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
 
 	// ===Object lifetime stuff===
 	private static final ReferenceQueue<IFullDataSource> lifeCycleDebugQueue = new ReferenceQueue<>();
+	private static final ReferenceQueue<IFullDataSource> softRefDebugQueue = new ReferenceQueue<>();
 	private static final Set<DataObjTracker> lifeCycleDebugSet = ConcurrentHashMap.newKeySet();
+	private static final Set<DataObjSoftTracker> softRefDebugSet = ConcurrentHashMap.newKeySet();
 	private static class DataObjTracker extends PhantomReference<IFullDataSource> implements Closeable
 	{
-		private final DhSectionPos pos;
+		public final DhSectionPos pos;
 		DataObjTracker(IFullDataSource data)
 		{
 			super(data, lifeCycleDebugQueue);
@@ -110,10 +117,21 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
 			lifeCycleDebugSet.add(this);
 			this.pos = data.getSectionPos();
 		}
-		
 		@Override
 		public void close() { lifeCycleDebugSet.remove(this); }
-		
+	}
+
+	private static class DataObjSoftTracker extends SoftReference<IFullDataSource> implements Closeable
+	{
+		public final FullDataMetaFile file;
+		DataObjSoftTracker(FullDataMetaFile file, IFullDataSource data)
+		{
+			super(data, softRefDebugQueue);
+			softRefDebugSet.add(this);
+			this.file = file;
+		}
+		@Override
+		public void close() { softRefDebugSet.remove(this); }
 	}
     // ===========================
 	
@@ -195,17 +213,18 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
 					// this exception can be ignored
 				}
 				else if (ex != null) {
-					LOGGER.error("Error loading file "+this.file+": ", ex);
+					LOGGER.error("Error updating file "+this.file+": ", ex);
 				}
 				if (fullDataSource != null) {
 					new DataObjTracker(fullDataSource);
+					new DataObjSoftTracker(this, fullDataSource);
 				}
 				//LOGGER.info("Updated file "+this.file);
 				if (pos.sectionDetailLevel == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL)
 					DebugRenderer.makeParticle(
 							new DebugRenderer.BoxParticle(
-									new DebugRenderer.Box(this.pos, 0, 256f, 0.05f, Color.green),
-									0.5, 512f
+									new DebugRenderer.Box(this.pos, 64f, 72f, 0.03f, Color.green.darker()),
+									0.2, 32f
 							)
 					);
 
@@ -421,7 +440,8 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
 		{
 			appendLock.unlock();
 		}
-		
+
+		this.flushAndSaveAsync();
 		//LOGGER.info("write queue length for pos "+this.pos+": " + writeQueue.queue.size());
 	}
 	
@@ -538,6 +558,14 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
 			phantom.close();
 			phantom = (DataObjTracker) lifeCycleDebugQueue.poll();
 		}
+
+		DataObjSoftTracker soft = (DataObjSoftTracker) softRefDebugQueue.poll();
+		while (soft != null)
+		{
+			//LOGGER.info("Full Data at pos: "+soft.file.pos+" has been soft released.");
+			soft.close();
+			soft = (DataObjSoftTracker) softRefDebugQueue.poll();
+		}
 	}
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java

@@ -76,7 +76,9 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 				continue;
 			}
 			metaFile.genQueueChecked = false; // unset it so it can be checked again
-			metaFile.markNeedUpdate();
+			if (data != null) {
+				metaFile.markNeedUpdate();
+			}
 		}
 		flushAndSave(); // Trigger an update to the meta files
 	}
@@ -143,8 +145,13 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 				worldGenQueue.submitGenTask(new DhLodPos(pos), dataSource.getDataDetailLevel(), genTask)
 						.whenComplete((genTaskResult, ex) ->
 						{
-							this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos);
-							this.fireOnGenPosSuccessListeners(pos);
+							if (genTaskResult.success) {
+								this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos);
+								this.fireOnGenPosSuccessListeners(pos);
+							}
+							else {
+								file.genQueueChecked = false;
+							}
 							this.incompleteDataSources.remove(pos);
 						});
 			}

core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java

@@ -79,6 +79,8 @@ public abstract class AbstractMetaDataContainerFile
     public final DhSectionPos pos;
 	
     public File file;
+
+	private volatile boolean DebugThreadCheck = false;
 	
 	
 	
@@ -190,6 +192,8 @@ public abstract class AbstractMetaDataContainerFile
 	
 	protected void writeData(IMetaDataWriterFunc<DhDataOutputStream> dataWriterFunc) throws IOException
 	{
+		LodUtil.assertTrue(!DebugThreadCheck);
+		DebugThreadCheck = true;
 		LodUtil.assertTrue(this.baseMetaData != null);
 		if (this.file.exists())
 		{
@@ -278,6 +282,7 @@ public abstract class AbstractMetaDataContainerFile
 			{
 				LOGGER.error(tempDeleteErrorMessage);
 			}
+			DebugThreadCheck = false;
 		}
 	}
 	

core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java

@@ -65,7 +65,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements
 		} else if (doesFileExist) {
 			c = Color.RED;
 		}
-		r.renderBox(new DebugRenderer.Box(pos, 0, 256, 0.05f, c));
+		r.renderBox(new DebugRenderer.Box(pos, 64, 72, 0.05f, c));
 	}
 
 	//=============//
@@ -121,16 +121,16 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements
 		renderSourceLoadFuture.thenAccept((renderSource) -> {
 			boolean worked = renderSource.fastWrite(chunkDataView, level);
 
-			if (pos.sectionDetailLevel == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL+5) {
+			//if (pos.sectionDetailLevel == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL+5) {
 				float offset = new Random(System.nanoTime() ^ Thread.currentThread().getId()).nextFloat() * 16f;
 				Color c = worked ? Color.blue : Color.red;
 				DebugRenderer.makeParticle(
 						new DebugRenderer.BoxParticle(
-								new DebugRenderer.Box(chunkDataView.getLodPos(), 0, 64f + offset, 0.07f, c),
+								new DebugRenderer.Box(chunkDataView.getLodPos(), 32f, 64f + offset, 0.07f, c),
 								2.0, 16f
 						)
 				);
-			}
+			//}
 		});
 	}
 	

core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java

@@ -11,6 +11,7 @@ import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
 import com.seibel.distanthorizons.core.dataObjects.transformers.DataRenderTransformer;
 import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
 import com.seibel.distanthorizons.core.level.IDhClientLevel;
+import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
 import com.seibel.distanthorizons.core.util.FileUtil;
 import com.seibel.distanthorizons.core.util.ThreadUtil;
 import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException;
@@ -18,6 +19,7 @@ import com.seibel.distanthorizons.core.config.Config;
 import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector;
 import org.apache.logging.log4j.Logger;
 
+import java.awt.*;
 import java.io.File;
 import java.io.IOException;
 import java.util.*;
@@ -293,14 +295,17 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 	//================//
 	// cache updating //
 	//================//
-	
+
     private CompletableFuture<Void> updateCacheAsync(ColumnRenderSource renderSource, RenderMetaDataFile file)
 	{
+		DebugRenderer.BoxWithLife box = new DebugRenderer.BoxWithLife(new DebugRenderer.Box(renderSource.sectionPos, 74f, 86f, 0.1f, Color.red), 1.0, 32f, Color.green.darker());
+
 		// get the full data source loading future
 		CompletableFuture<IFullDataSource> fullDataSourceFuture = this.fullDataSourceProvider.read(renderSource.getSectionPos());
 		fullDataSourceFuture = fullDataSourceFuture.thenApply((fullDataSource) -> 
 			{
 				// the fullDataSource can be null if the thread this was running on was interrupted
+				box.box.color = Color.yellow.darker();
 				return fullDataSource;
 			}).exceptionally((ex) -> 
 			{
@@ -341,6 +346,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 							LOGGER.error("Exception when updating render file using data source: ", ex);
 						}
 					}
+					box.close();
 					transformationCompleteFuture.complete(null);
 				});
 		return transformationCompleteFuture;

core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java

@@ -40,7 +40,8 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 	private final IDhApiWorldGenerator generator;
 	
 	/** contains the positions that need to be generated */
-	private final QuadTree<WorldGenTask> waitingTaskQuadTree;
+	//private final QuadTree<WorldGenTask> waitingTaskQuadTree;
+	private final ConcurrentHashMap<DhLodPos, WorldGenTask> waitingTasks = new ConcurrentHashMap<>();
 	
 	private final ConcurrentHashMap<DhLodPos, InProgressWorldGenTaskGroup> inProgressGenTasksByLodPos = new ConcurrentHashMap<>();
 	
@@ -94,7 +95,7 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 		//FIXME: Currently resizing view dist doesn't update this, causing some gen task to fail.
 		int treeWidth = Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistance.get() * LodUtil.CHUNK_WIDTH * 2; // TODO the *2 is to allow for generation edge cases, and should probably be removed at some point
 		byte treeMinDetailLevel = LodUtil.CHUNK_DETAIL_LEVEL; // The min level should be at least fill in 1 ChunkSizedFullDataAccessor.
-		this.waitingTaskQuadTree = new QuadTree<>(treeWidth, DhBlockPos2D.ZERO /*the quad tree will be re-centered later*/, treeMinDetailLevel);
+		//this.waitingTaskQuadTree = new QuadTree<>(treeWidth, DhBlockPos2D.ZERO /*the quad tree will be re-centered later*/, treeMinDetailLevel);
 		
 		
 		if (this.minGranularity < LodUtil.CHUNK_DETAIL_LEVEL)
@@ -139,16 +140,19 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 		LodUtil.assertTrue(pos.detailLevel > requiredDataDetail + LodUtil.CHUNK_DETAIL_LEVEL);
 
 		DhSectionPos requestPos = new DhSectionPos(pos.detailLevel, pos.x, pos.z);
-		if (this.waitingTaskQuadTree.isSectionPosInBounds(requestPos))
+
+
+		//if (this.waitingTaskQuadTree.isSectionPosInBounds(requestPos))
 		{
 			CompletableFuture<WorldGenResult> future = new CompletableFuture<>();
-			this.waitingTaskQuadTree.setValue(requestPos, new WorldGenTask(pos, requiredDataDetail, tracker, future));
+			//this.waitingTaskQuadTree.setValue(requestPos, new WorldGenTask(pos, requiredDataDetail, tracker, future));
+			waitingTasks.put(pos, new WorldGenTask(pos, requiredDataDetail, tracker, future));
 			return future;
 		}
-		else
-		{
-			return CompletableFuture.completedFuture(WorldGenResult.CreateFail());
-		}
+		//else
+		//{
+			//return CompletableFuture.completedFuture(WorldGenResult.CreateFail());
+		//}
 	}
 	
 	
@@ -197,7 +201,7 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 //					LOGGER.info("pre task count: " + this.numberOfTasksQueued);
 					
 					// recenter the generator tasks, this is done to prevent generating chunks where the player isn't
-					this.waitingTaskQuadTree.setCenterBlockPos(this.generationTargetPos);
+					//this.waitingTaskQuadTree.setCenterBlockPos(this.generationTargetPos);
 					
 					// queue generation tasks until the generator is full, or there are no more tasks to generate
 					boolean taskStarted = true;
@@ -252,6 +256,15 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 
 	private final Set<WorldGenTask> CheckingTasks = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
+	private static class Mapper {
+		public final WorldGenTask task;
+		public final int dist;
+		public Mapper(WorldGenTask task, int dist) {
+			this.task = task;
+			this.dist = dist;
+		}
+	}
+
 	/** 
 	 * @param targetPos the position to center the generation around 
 	 * @return false if no tasks were found to generate
@@ -261,9 +274,9 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 		long closestGenDist = Long.MAX_VALUE;
 		
 		WorldGenTask closestTask = null;
-		CheckingTasks.clear();
+		//CheckingTasks.clear();
 		
-		// TODO improve, having to go over every node isn't super efficient, removing null nodes from the tree would help
+/*		// TODO improve, having to go over every node isn't super efficient, removing null nodes from the tree would help
 		Iterator<QuadNode<WorldGenTask>> nodeIterator = this.waitingTaskQuadTree.nodeIterator();
 		while (nodeIterator.hasNext())
 		{
@@ -291,16 +304,28 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 					closestGenDist = chebDistToTargetPos;
 				}
 			}
-		}
-		
-		if (closestTask == null)
-		{
-			// no task was found, this probably means there isn't anything left to generate
+		}*/
+
+		waitingTasks.forEach((pos, task) -> {
+			if (!task.StillValid()) {
+				waitingTasks.remove(pos);
+				task.future.complete(WorldGenResult.CreateFail());
+			}
+		});
+
+		if (waitingTasks.size() == 0) {
 			return false;
 		}
+
+		Mapper closestTaskMap = waitingTasks.reduceEntries(1024,
+				v -> new Mapper(v.getValue(), v.getValue().pos.getCenterBlockPos().toPos2D().chebyshevDist(targetPos.toPos2D())),
+				(a, b) -> a.dist < b.dist ? a : b);
+
+		closestTask = closestTaskMap.task;
 		
 		// remove the task we found, we are going to start it and don't want to run it multiple times
-		WorldGenTask removedWorldGenTask = this.waitingTaskQuadTree.setValue(new DhSectionPos(closestTask.pos.detailLevel, closestTask.pos.x, closestTask.pos.z), null);
+		//WorldGenTask removedWorldGenTask = this.waitingTaskQuadTree.setValue(new DhSectionPos(closestTask.pos.detailLevel, closestTask.pos.x, closestTask.pos.z), null);
+		waitingTasks.remove(closestTask.pos, closestTask);
 		
 		// do we need to modify this task to generate it?
 		if(this.canGeneratePos((byte) 0, closestTask.pos)) // TODO should detail level 0 be replaced?
@@ -339,24 +364,25 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 			// split up the task and add each one to the tree
 			LinkedList<CompletableFuture<WorldGenResult>> childFutures = new LinkedList<>();
 			DhSectionPos sectionPos = new DhSectionPos(closestTask.pos.detailLevel, closestTask.pos.x, closestTask.pos.z);
-			sectionPos.forEachChild((childDhSectionPos) -> 
+			WorldGenTask finalClosestTask = closestTask;
+			sectionPos.forEachChild((childDhSectionPos) ->
 			{
 				CompletableFuture<WorldGenResult> newFuture = new CompletableFuture<>();
 				childFutures.add(newFuture);
 				
-				WorldGenTask newGenTask = new WorldGenTask(new DhLodPos(childDhSectionPos.sectionDetailLevel, childDhSectionPos.sectionX, childDhSectionPos.sectionZ), childDhSectionPos.sectionDetailLevel, removedWorldGenTask.taskTracker, newFuture);
-				this.waitingTaskQuadTree.setValue(new DhSectionPos(childDhSectionPos.sectionDetailLevel, childDhSectionPos.sectionX, childDhSectionPos.sectionZ), newGenTask);
+				WorldGenTask newGenTask = new WorldGenTask(new DhLodPos(childDhSectionPos.sectionDetailLevel, childDhSectionPos.sectionX, childDhSectionPos.sectionZ), childDhSectionPos.sectionDetailLevel, finalClosestTask.taskTracker, newFuture);
+				waitingTasks.put(newGenTask.pos, newGenTask);
+				//this.waitingTaskQuadTree.setValue(new DhSectionPos(childDhSectionPos.sectionDetailLevel, childDhSectionPos.sectionX, childDhSectionPos.sectionZ), newGenTask);
 				
-				boolean valueAdded = this.waitingTaskQuadTree.getValue(new DhSectionPos(childDhSectionPos.sectionDetailLevel, childDhSectionPos.sectionX, childDhSectionPos.sectionZ)) != null;
-				LodUtil.assertTrue(valueAdded); // failed to add world gen task to quad tree, this means the quad tree was the wrong size
+				//boolean valueAdded = this.waitingTaskQuadTree.getValue(new DhSectionPos(childDhSectionPos.sectionDetailLevel, childDhSectionPos.sectionX, childDhSectionPos.sectionZ)) != null;
+				//LodUtil.assertTrue(valueAdded); // failed to add world gen task to quad tree, this means the quad tree was the wrong size
 				
 //				LOGGER.info("split feature "+sectionPos+" into "+childDhSectionPos+" "+(valueAdded ? "added" : "notAdded"));
 			});
 			
 			// send the child futures to the future recipient, to notify them of the new tasks
-			removedWorldGenTask.future.complete(WorldGenResult.CreateSplit(childFutures));
-			
-			
+			closestTask.future.complete(WorldGenResult.CreateSplit(childFutures));
+
 			// return true so we attempt to generate again
 			return true;
 		}
@@ -640,8 +666,8 @@ public class WorldGenerationQueue implements Closeable, IDebugRenderable
 	@Override
 	public void debugRender(DebugRenderer r) {
 		//if (true) return;
-		CheckingTasks.forEach((t) -> {
-			DhLodPos pos = t.pos;
+		waitingTasks.keySet().forEach((pos) -> {
+			//DhLodPos pos = t.pos;
 			r.renderBox(new DebugRenderer.Box(pos, -32f, 64f, 0.05f, Color.blue));
 		});
 		this.inProgressGenTasksByLodPos.forEach((pos, t) -> {

core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java

@@ -178,11 +178,11 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS
 	@Override 
 	public void onWorldGenTaskComplete(DhSectionPos pos)
 	{
-		if (pos.sectionDetailLevel == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL)
+		//if (pos.sectionDetailLevel == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL)
 			DebugRenderer.makeParticle(
 					new DebugRenderer.BoxParticle(
-							new DebugRenderer.Box(pos, 0, 256f, 0.09f, Color.red),
-							0.5, 512f
+							new DebugRenderer.Box(pos, 128f, 156f, 0.09f, Color.red.darker()),
+							0.2, 32f
 					)
 			);
 		clientside.reloadPos(pos);

core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java

@@ -267,8 +267,8 @@ public class LodRenderSection implements IDebugRenderable
 			if (pos.sectionDetailLevel == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL)
 				DebugRenderer.makeParticle(
 						new DebugRenderer.BoxParticle(
-								new DebugRenderer.Box(pos, 0, 256f, -0.1f, Color.yellow),
-								1.0, 512f
+								new DebugRenderer.Box(pos, 32f, 64f, 0.2f, Color.yellow),
+								0.5, 16f
 						)
 				);
 			neighborUpdated = false;

core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java

@@ -24,6 +24,8 @@ import org.jetbrains.annotations.Nullable;
 import org.lwjgl.opengl.GL32;
 
 import java.awt.*;
+import java.io.Closeable;
+import java.io.IOException;
 import java.lang.ref.WeakReference;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -71,9 +73,9 @@ public class DebugRenderer {
     };
 
     public static final class Box {
-        public final Vec3f a;
-        public final Vec3f b;
-        public final Color color;
+        public Vec3f a;
+        public Vec3f b;
+        public Color color;
 
         public Box(Vec3f a, Vec3f b, Color color) {
             this.a = a;
@@ -130,10 +132,10 @@ public class DebugRenderer {
     private final LinkedList<WeakReference<IDebugRenderable>> renderers = new LinkedList<>();
 
     public static final class BoxParticle implements Comparable<BoxParticle> {
-        private final Box box;
-        private final long startTime;
-        private final long duration;
-        private final float yChange;
+        public Box box;
+        public long startTime;
+        public long duration;
+        public float yChange;
 
         public BoxParticle(Box box, long startTime, long duration, float yChange) {
             this.box = box;
@@ -152,7 +154,7 @@ public class DebugRenderer {
 
         @Override
         public int compareTo(@NotNull DebugRenderer.BoxParticle o) {
-            return Long.compare(startTime, o.startTime);
+            return Long.compare(startTime + duration, o.startTime + o.duration);
         }
 
         Box getBox() {
@@ -168,6 +170,41 @@ public class DebugRenderer {
         }
     }
 
+    public static final class BoxWithLife implements IDebugRenderable, Closeable {
+        public Box box;
+        public BoxParticle particaleOnClose;
+
+        public BoxWithLife(Box box, long ns, float yChange, Color deathColor) {
+            this.box = box;
+            this.particaleOnClose = new BoxParticle(new Box(box.a, box.b, deathColor), -1, ns, yChange);
+            DebugRenderer.register(this);
+        }
+
+        public BoxWithLife(Box box, long ns, float yChange) {
+            this(box, ns, yChange, box.color);
+        }
+
+        public BoxWithLife(Box box, double s, float yChange, Color deathColor) {
+            this.box = box;
+            this.particaleOnClose = new BoxParticle(new Box(box.a, box.b, deathColor), s, yChange);
+        }
+
+        public BoxWithLife(Box box, double s, float yChange) {
+            this(box, s, yChange, box.color);
+        }
+
+        @Override
+        public void debugRender(DebugRenderer r) {
+            r.renderBox(box);
+        }
+
+        @Override
+        public void close() {
+            makeParticle(new BoxParticle(particaleOnClose.getBox(), System.nanoTime(), particaleOnClose.duration, particaleOnClose.yChange));
+            DebugRenderer.unregister(this);
+        }
+    }
+
     private final PriorityBlockingQueue<BoxParticle> particles = new PriorityBlockingQueue<>();
 
     public static void unregister(IDebugRenderable r) {