Potentially fix duplicate world generator issue?

core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java

@@ -255,26 +255,6 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
 	@Override
 	public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.get(relativeX, relativeZ); }
 	
-	@Override
-	public ArrayList<DhSectionPos> getUngeneratedPosList()
-	{
-		ArrayList<DhSectionPos> posList = new ArrayList<>();
-		
-		for (int x = 0; x < this.width; x++)
-		{
-			for (int z = 0; z < this.width; z++)
-			{
-				SingleColumnFullDataAccessor column = this.get(x,z);
-				if (column == null || !column.doesColumnExist())
-				{
-					posList.add(new DhSectionPos(this.sectionPos.sectionDetailLevel, this.sectionPos.sectionX + x, this.sectionPos.sectionZ + z));
-				}
-			}
-		}
-		
-		return posList;
-	}
-	
 	@Override
 	public void update(ChunkSizedFullDataAccessor chunkDataView)
 	{

core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java

@@ -392,26 +392,6 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
 		return chunk.get(relativeX % this.dataPointsPerSection, relativeZ % this.dataPointsPerSection);
 	}
 	
-	@Override
-	public ArrayList<DhSectionPos> getUngeneratedPosList()
-	{
-		ArrayList<DhSectionPos> posList = new ArrayList<>();
-		
-		for (int x = 0; x < SECTION_SIZE; x++)
-		{
-			for (int z = 0; z < SECTION_SIZE; z++)
-			{
-				SingleColumnFullDataAccessor column = this.tryGet(x,z);
-				if (column == null || !column.doesColumnExist())
-				{
-					posList.add(new DhSectionPos(this.sectionPos.sectionDetailLevel, this.sectionPos.sectionX + x, this.sectionPos.sectionZ + z));
-				}
-			}
-		}
-		
-		return posList;
-	}
-	
 	
 	
 	//=========//

core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java

@@ -265,26 +265,6 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
     @Override
     public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.isColumnNotEmpty.get(relativeX * WIDTH + relativeZ) ? this.get(relativeX, relativeZ) : null; }
 	
-	@Override
-	public ArrayList<DhSectionPos> getUngeneratedPosList()
-	{
-		ArrayList<DhSectionPos> posList = new ArrayList<>();
-		
-		for (int x = 0; x < this.width; x++)
-		{
-			for (int z = 0; z < this.width; z++)
-			{
-				SingleColumnFullDataAccessor column = this.get(x,z);
-				if (column == null || !column.doesColumnExist())
-				{
-					posList.add(new DhSectionPos(this.sectionPos.sectionDetailLevel, this.sectionPos.sectionX + x, this.sectionPos.sectionZ + z));
-				}
-			}
-		}
-		
-		return posList;
-	}
-	
 	
 	
 	//=====================//

core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java

@@ -5,15 +5,18 @@ import com.seibel.lod.core.dataObjects.fullData.FullDataPointIdMap;
 import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
 import com.seibel.lod.core.dataObjects.fullData.accessor.IFullDataAccessor;
 import com.seibel.lod.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor;
+import com.seibel.lod.core.dataObjects.fullData.sources.CompleteFullDataSource;
 import com.seibel.lod.core.dataObjects.render.ColumnRenderSource;
 import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
 import com.seibel.lod.core.level.IDhLevel;
 import com.seibel.lod.core.pos.DhSectionPos;
 import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
 import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream;
+import com.seibel.lod.coreapi.util.BitShiftUtil;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedList;
 
 /**
  * Base for all Full Data Source objects. <br><br>
@@ -65,10 +68,185 @@ public interface IFullDataSource
 	
 	FullDataPointIdMap getMapping();
 	
-	/** @return true if every datapoint in this object has been generated, false otherwise. */
-	default boolean isCompletelyGenerated() { return this.getUngeneratedPosList().size() == 0; }
-	/** @return the list of {@link DhSectionPos} that aren't generated in this data source. */
-	ArrayList<DhSectionPos> getUngeneratedPosList();
+	/** 
+	 * @param highestGeneratorDetailLevel the smallest numerical detail level that the un-generated positions should be split into
+	 * @return the list of {@link DhSectionPos} that aren't generated in this data source. 
+	 */
+	default ArrayList<DhSectionPos> getUngeneratedPosList(byte highestGeneratorDetailLevel, boolean onlyReturnPositionsTheGeneratorCanAccept) 
+	{
+		ArrayList<DhSectionPos> posArray = this.getUngeneratedPosList(this.getSectionPos(), highestGeneratorDetailLevel);
+		
+		if (onlyReturnPositionsTheGeneratorCanAccept)
+		{
+			LinkedList<DhSectionPos> posList = new LinkedList<>(posArray);
+			
+			ArrayList<DhSectionPos> cleanedPosArray = new ArrayList<>();
+			while (posList.size() > 0)
+			{
+				DhSectionPos pos = posList.remove();
+				if (pos.sectionDetailLevel > highestGeneratorDetailLevel)
+				{
+					pos.forEachChild((childPos) -> { posList.push(childPos); });
+				}
+				else
+				{
+					cleanedPosArray.add(pos);
+				}
+			}
+			
+			return cleanedPosArray;
+		}
+		else
+		{
+			return posArray;
+		}
+	}
+	default ArrayList<DhSectionPos> getUngeneratedPosList(DhSectionPos quadrantPos, byte highestGeneratorDetailLevel)
+	{
+		ArrayList<DhSectionPos> ungeneratedPosList = new ArrayList<>();
+		
+		int sourceRelWidth = this.getWidthInDataPoints();
+		
+		
+		if (quadrantPos.sectionDetailLevel < highestGeneratorDetailLevel)
+		{
+			throw new IllegalArgumentException("detail level lower than world generator can accept.");
+		}
+		else if (quadrantPos.sectionDetailLevel == highestGeneratorDetailLevel)
+		{
+			// we are at the highest detail level the world generator can accept,
+			// we either need to generate this whole section, or not at all
+			
+			// TODO combine duplicate code
+			
+			byte childDetailLevel = (byte) (quadrantPos.sectionDetailLevel);
+			
+			int quadrantDetailLevelDiff = this.getSectionPos().sectionDetailLevel - childDetailLevel;
+			int widthInSecPos = BitShiftUtil.powerOfTwo(quadrantDetailLevelDiff);
+			int relWidthForSecPos = sourceRelWidth / widthInSecPos;
+			
+			DhSectionPos minSecPos = this.getSectionPos().convertToDetailLevel(childDetailLevel);
+			DhSectionPos inputPos = quadrantPos;
+			
+			
+			
+			int minRelX = inputPos.sectionX - minSecPos.sectionX;
+			int minRelZ = inputPos.sectionZ - minSecPos.sectionZ;
+			int maxRelX = minRelX + 1;
+			int maxRelZ = minRelZ + 1;
+			
+			minRelX = minRelX * relWidthForSecPos;
+			minRelZ = minRelZ * relWidthForSecPos;
+			maxRelX = maxRelX * relWidthForSecPos;
+			maxRelZ = maxRelZ * relWidthForSecPos;
+			
+			if (this.getClass() != CompleteFullDataSource.class)
+			{
+				int breakpoint= 0;
+			}
+			
+			
+			boolean quadrantFullyGenerated = true;
+			for (int relX = minRelX; relX < maxRelX; relX++)
+			{
+				for (int relZ = minRelZ; relZ < maxRelZ; relZ++)
+				{
+					SingleColumnFullDataAccessor column = this.tryGet(relX, relZ);
+					if (column == null)
+					{
+						int breakpoi= 0;
+					}
+					
+					if (column == null || !column.doesColumnExist())// || column.hasNullDataPoints())
+					{
+						// no data for this relative position
+						quadrantFullyGenerated = false;
+						break;
+					}
+				}
+			}
+
+			if (!quadrantFullyGenerated)
+			{
+				// at least 1 data point is missing,
+				// this whole section must be regenerated
+				ungeneratedPosList.add(quadrantPos);
+			}
+		}
+		else
+		{
+			// TODO comment
+			// TODO combine duplicate code
+			
+			byte childDetailLevel = (byte) (quadrantPos.sectionDetailLevel-1);
+			
+			for (int i = 0; i < 4; i++)
+			{
+				int quadrantDetailLevelDiff = this.getSectionPos().sectionDetailLevel - childDetailLevel;
+				int widthInSecPos = BitShiftUtil.powerOfTwo(quadrantDetailLevelDiff);
+				int relWidthForSecPos = sourceRelWidth / widthInSecPos;
+				
+				DhSectionPos minSecPos = this.getSectionPos().convertToDetailLevel(childDetailLevel);
+				DhSectionPos inputPos = quadrantPos.getChildByIndex(i);
+				
+				
+				
+				int minRelX = inputPos.sectionX - minSecPos.sectionX;
+				int minRelZ = inputPos.sectionZ - minSecPos.sectionZ;
+				int maxRelX = minRelX + 1;
+				int maxRelZ = minRelZ + 1;
+				
+				minRelX = minRelX * relWidthForSecPos;
+				minRelZ = minRelZ * relWidthForSecPos;
+				maxRelX = maxRelX * relWidthForSecPos;
+				maxRelZ = maxRelZ * relWidthForSecPos;
+				
+				
+				
+				boolean quadrantFullyGenerated = true;
+				boolean quadrantEmpty = true;
+				for (int relX = minRelX; relX < maxRelX; relX++)
+				{
+					for (int relZ = minRelZ; relZ < maxRelZ; relZ++)
+					{
+						SingleColumnFullDataAccessor column = this.tryGet(relX, relZ);
+						if (column == null || !column.doesColumnExist()) // || column.hasNullDataPoints())
+						{
+							// no data for this relative position
+							quadrantFullyGenerated = false;
+						}
+						else
+						{
+							// data exists for this pos
+							quadrantEmpty = false;
+						}
+					}
+				}
+				
+				
+				if (quadrantFullyGenerated)
+				{
+					// no generation necessary
+					continue;
+				}
+				else if (quadrantEmpty)
+				{
+					// nothing exists for this sub quadrant, add this sub-quadrant's position
+					ungeneratedPosList.add(inputPos);
+				}
+				else
+				{
+					// some data exists in this quadrant, but not all that we need
+					// recurse down to determine which sub-quadrant positions will need generation
+					
+					ungeneratedPosList.addAll(this.getUngeneratedPosList(inputPos, highestGeneratorDetailLevel));
+				}
+				
+			}
+		}
+		
+		return ungeneratedPosList;
+	}
 	
 	
 	

core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java

@@ -30,14 +30,14 @@ import java.util.function.Function;
 public class FullDataFileHandler implements IFullDataSourceProvider
 {
     private static final Logger LOGGER = DhLoggerBuilder.getLogger();
-    
+	
 	// TODO add config option to set pool size
-	final ExecutorService fileHandlerThread = ThreadUtil.makeThreadPool(4, FullDataFileHandler.class.getSimpleName()+"Thread");
-    final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> files = new ConcurrentHashMap<>();
-    final IDhLevel level;
-    final File saveDir;
-    AtomicInteger topDetailLevel = new AtomicInteger(-1);
-    final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET;
+	protected final ExecutorService fileHandlerThread = ThreadUtil.makeThreadPool(4, FullDataFileHandler.class.getSimpleName()+"Thread");
+	protected final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> fileBySectionPos = new ConcurrentHashMap<>();
+	protected final IDhLevel level;
+	protected final File saveDir;
+	protected final AtomicInteger topDetailLevel = new AtomicInteger(-1);
+	protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET;
 	
 	
 	
@@ -130,13 +130,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider
             }
             // Add file to the list of files.
 			this.topDetailLevel.updateAndGet(v -> Math.max(v, fileToUse.pos.sectionDetailLevel));
-			this.files.put(pos, fileToUse);
+			this.fileBySectionPos.put(pos, fileToUse);
         }
     }
 	
     protected FullDataMetaFile getOrMakeFile(DhSectionPos pos)
 	{
-        FullDataMetaFile metaFile = this.files.get(pos);
+        FullDataMetaFile metaFile = this.fileBySectionPos.get(pos);
         if (metaFile == null)
 		{
             FullDataMetaFile newMetaFile;
@@ -149,7 +149,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
                 LOGGER.error("IOException on creating new data file at {}", pos, e);
                 return null;
             }
-            metaFile = this.files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
+            metaFile = this.fileBySectionPos.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
             if (metaFile == null) 
 			{
 				metaFile = newMetaFile;
@@ -190,7 +190,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
 						continue;
 					}
 					
-                    if (this.files.containsKey(subPos))
+                    if (this.fileBySectionPos.containsKey(subPos))
 					{
                         allEmpty = false;
                         break outerLoop;
@@ -219,7 +219,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
 		DhSectionPos childPos = pos.getChildByIndex(childIndex);
 		if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos))
 		{
-			FullDataMetaFile metaFile = this.files.get(childPos);
+			FullDataMetaFile metaFile = this.fileBySectionPos.get(childPos);
 			if (metaFile != null)
 			{
 				// we have reached a populated leaf node in the quad tree
@@ -285,7 +285,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
     }
     private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData)
 	{
-        FullDataMetaFile metaFile = this.files.get(sectionPos);
+        FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos);
         if (metaFile != null)
 		{ 
 			// there is a file for this position
@@ -304,7 +304,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
     public CompletableFuture<Void> flushAndSave()
 	{
         ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
-        for (FullDataMetaFile metaFile : this.files.values())
+        for (FullDataMetaFile metaFile : this.fileBySectionPos.values())
 		{
             futures.add(metaFile.flushAndSaveAsync());
         }
@@ -392,7 +392,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
 		
 		FileUtil.renameCorruptedFile(metaFile.file);
 		// remove the FullDataMetaFile since the old one was corrupted
-		this.files.remove(pos);
+		this.fileBySectionPos.remove(pos);
 		// create a new FullDataMetaFile to write new data to
 		return this.getOrMakeFile(pos);
 	}

core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java

@@ -344,7 +344,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile
 						{
 							if (exception != null)
 							{
-								LOGGER.error("Error refreshing data "+this.pos+": "+exception+" "+exception.getMessage());
+								LOGGER.error("Error refreshing data "+this.pos+": "+exception+" "+exception.getMessage(), exception);
 								future.complete(null);
 								this.cachedFullDataSource = new SoftReference<>(null);
 							}

core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java

@@ -9,6 +9,7 @@ import com.seibel.lod.core.generation.tasks.IWorldGenTaskTracker;
 import com.seibel.lod.core.generation.WorldGenerationQueue;
 import com.seibel.lod.core.generation.tasks.WorldGenResult;
 import com.seibel.lod.core.level.IDhServerLevel;
+import com.seibel.lod.core.pos.DhLodPos;
 import com.seibel.lod.core.pos.DhSectionPos;
 import com.seibel.lod.core.logging.DhLoggerBuilder;
 import com.seibel.lod.core.util.LodUtil;
@@ -24,21 +25,19 @@ import java.util.function.Consumer;
 
 public class GeneratedFullDataFileHandler extends FullDataFileHandler
 {
-    private static final Logger LOGGER = DhLoggerBuilder.getLogger();
+	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	
-    private final AtomicReference<WorldGenerationQueue> worldGenQueueRef = new AtomicReference<>(null);
+	private final AtomicReference<WorldGenerationQueue> worldGenQueueRef = new AtomicReference<>(null);
 	
 	private final ArrayList<IOnWorldGenCompleteListener> onWorldGenTaskCompleteListeners = new ArrayList<>();
 	
-	/** 
+	/**
 	 * Keeps track of which partially generated {@link IFullDataSource} {@link DhSectionPos}' are waiting to be generated.
 	 * This is done to prevent sending duplicate generation requests for the same position.
 	 */
 	private final HashSet<DhSectionPos> incompleteSourceGenRequests = new HashSet<>();
 	
-	
-	
-    public GeneratedFullDataFileHandler(IDhServerLevel level, File saveRootDir) { super(level, saveRootDir); }
+	public GeneratedFullDataFileHandler(IDhServerLevel level, File saveRootDir) { super(level, saveRootDir); }
 	
 	
 	
@@ -46,11 +45,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 	// data //
 	//======//
 	
+	@Override
 	public CompletableFuture<IFullDataSource> read(DhSectionPos pos)
 	{
-		return super.read(pos).whenComplete((fullDataSource, ex) -> 
+		return super.read(pos).whenComplete((fullDataSource, ex) ->
 		{
-			this.checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource);
+			this.DISABLED_checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource);
 		});
 	}
 	
@@ -61,11 +61,11 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 	//==================//
 	
 	/** Assumes there isn't a pre-existing queue. */
-    public void setGenerationQueue(WorldGenerationQueue newWorldGenQueue)
+	public void setGenerationQueue(WorldGenerationQueue newWorldGenQueue)
 	{
-        boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue);
-        LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!");
-    }
+		boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue);
+		LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!");
+	}
 	
 	public void clearGenerationQueue() { this.worldGenQueueRef.set(null); }
 	
@@ -76,6 +76,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 	//=================//
 	
 	public void addWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.add(listener); }
+	
 	public void removeWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.remove(listener); }
 	
 	
@@ -84,17 +85,17 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 	// events //
 	//========//
 	
-    @Override
-    public CompletableFuture<IFullDataSource> onCreateDataFile(FullDataMetaFile file)
+	@Override
+	public CompletableFuture<IFullDataSource> onCreateDataFile(FullDataMetaFile file)
 	{
-        DhSectionPos pos = file.pos;
-        
+		DhSectionPos pos = file.pos;
+		
 		ArrayList<FullDataMetaFile> existingFiles = new ArrayList<>();
-        ArrayList<DhSectionPos> missingPositions = new ArrayList<>();
+		ArrayList<DhSectionPos> missingPositions = new ArrayList<>();
 		this.getDataFilesForPosition(pos, pos, existingFiles, missingPositions);
 		
 		// confirm the quad tree has at least one node in it
-        LodUtil.assertTrue(!missingPositions.isEmpty() || !existingFiles.isEmpty());
+		LodUtil.assertTrue(!missingPositions.isEmpty() || !existingFiles.isEmpty());
 		
 		
 		// determine the type of dataSource that should be used for this position
@@ -109,76 +110,84 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 		}
 		
 		
-        if (missingPositions.size() == 1 && existingFiles.isEmpty() && missingPositions.get(0).equals(pos))
+		if (missingPositions.size() == 1 && existingFiles.isEmpty() && missingPositions.get(0).equals(pos))
 		{
-            // No LOD data exists for this position yet
+			// No LOD data exists for this position yet
 			
-            WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
-            if (worldGenQueue != null)
+			WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
+			if (worldGenQueue != null)
 			{
+				this.incompleteSourceGenRequests.add(pos);
+				
 				// queue this section to be generated
 				GenTask genTask = new GenTask(pos, new WeakReference<>(incompleteFullDataSource));
-				worldGenQueue.submitGenTask(incompleteFullDataSource.getSectionPos().getSectionBBoxPos(), incompleteFullDataSource.getDataDetailLevel(), genTask)
-							 .whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos));
-            }
+				worldGenQueue.submitGenTask(new DhLodPos(pos), incompleteFullDataSource.getDataDetailLevel(), genTask)
+						.whenComplete((genTaskResult, ex) ->
+						{
+							this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos);
+							this.fireOnGenPosSuccessListeners(pos);
+							this.incompleteSourceGenRequests.remove(pos);
+						});
+			}
 			
 			// return the empty dataSource (it will be populated later)
-            return CompletableFuture.completedFuture(incompleteFullDataSource);
-        }
+			return CompletableFuture.completedFuture(incompleteFullDataSource);
+		}
 		else
 		{
 			// LOD data exists for this position
 			
 			// create the missing metaData files
-            for (DhSectionPos missingPos : missingPositions)
+			for (DhSectionPos missingPos : missingPositions)
 			{
-                FullDataMetaFile newFile = this.getOrMakeFile(missingPos);
-                if (newFile != null)
+				FullDataMetaFile newFile = this.getOrMakeFile(missingPos);
+				if (newFile != null)
 				{
 					existingFiles.add(newFile);
 				}
-            }
+			}
 			
-//            LOGGER.debug("Creating "+pos+" from sampling "+existingFiles.size()+" files: "+existingFiles);
+			//            LOGGER.debug("Creating "+pos+" from sampling "+existingFiles.size()+" files: "+existingFiles);
 			
 			// read in the existing data
 			final ArrayList<CompletableFuture<Void>> loadDataFutures = new ArrayList<>(existingFiles.size());
-            for (FullDataMetaFile existingFile : existingFiles)
+			for (FullDataMetaFile existingFile : existingFiles)
 			{
-                loadDataFutures.add(existingFile.loadOrGetCachedDataSourceAsync()
-					.exceptionally((ex) -> /*Ignore file read errors*/null)
-					.thenAccept((fullDataSource) ->
-					{
-						if (fullDataSource == null)
+				loadDataFutures.add(existingFile.loadOrGetCachedDataSourceAsync()
+						.exceptionally((ex) -> /*Ignore file read errors*/null)
+						.thenAccept((fullDataSource) ->
 						{
-							return;
-						}
-						
-						this.checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource);
-						
-						//LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos);
-						incompleteFullDataSource.sampleFrom(fullDataSource);
-					})
-                );
-            }
+							if (fullDataSource == null)
+							{
+								return;
+							}
+							
+							this.DISABLED_checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource);
+							
+							//LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos);
+							incompleteFullDataSource.sampleFrom(fullDataSource);
+						})
+				);
+			}
 			
-            return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0]))
-                    .thenApply((voidValue) -> incompleteFullDataSource.tryPromotingToCompleteDataSource());
-        }
-    }
-	
-	/** 
-	 * Checks if the given {@link IFullDataSource} is fully generated and
-	 * if it isn't, new world gen request(s) will be created.
-	 */
-	private void checkIfSectionNeedsAdditionalGeneration(DhSectionPos pos, IFullDataSource fullDataSource)
-	{
-		boolean generateSection = fullDataSource == null || (!fullDataSource.isCompletelyGenerated() && !incompleteSourceGenRequests.contains(pos));
-		if (!generateSection)
-		{
-			// this section doesn't need to be generated
-			return;
+			return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0]))
+					.thenApply((voidValue) -> incompleteFullDataSource.tryPromotingToCompleteDataSource());
 		}
+	}
+	
+	/**
+	 * Checks if the given {@link IFullDataSource} is fully generated and
+	 * if it isn't, creates the necessary world gen request(s) to finish it. <br>
+	 * Should be used to fill out partially generated {@link IFullDataSource}'s,
+	 * not populate empty ones.
+	 */
+	private void DISABLED_checkIfSectionNeedsAdditionalGeneration(DhSectionPos pos, IFullDataSource fullDataSource)
+	{
+		// FIXME method is disabled since it will often cause duplicate world gen requests which cause other issues
+		//  and James is out of ideas for how to fix it.
+		if (true)
+			return;
+		
 		
 		WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
 		if (worldGenQueue == null)
@@ -188,34 +197,63 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 		}
 		
 		
-		// the data source could be null if no file exists for this position
-		if (fullDataSource == null)
+		if (fullDataSource == null || fullDataSource.isEmpty())
 		{
-			if (pos.sectionDetailLevel <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL)
-			{
-				fullDataSource = HighDetailIncompleteFullDataSource.createEmpty(pos);
-			}
-			else
-			{
-				fullDataSource = LowDetailIncompleteFullDataSource.createEmpty(pos);
-			}
+			// none of the data source has been generated, those generation requests should be handled elsewhere
+			return;
+		}
+		else if (this.incompleteSourceGenRequests.contains(pos))
+		{
+			return;
+		}
+		
+		ArrayList<DhSectionPos> ungeneratedPosList = fullDataSource.getUngeneratedPosList(worldGenQueue.maxGranularity, true);
+		if (ungeneratedPosList.size() == 0)
+		{
+			// this section doesn't need to be generated
+			return;
 		}
 		
 		
 		
-		incompleteSourceGenRequests.add(pos);
-		//LOGGER.info("["+ungeneratedPosList.size()+"] missing sub positions for pos: ["+pos+"]. Number of gen requests queued: ["+queuedGenRequests.size()+"].");
-
-		// note: this will potentially re-generate terrain, however due to the generator setup this is currently unavoidable and probably not worth worrying about
-		GenTask genTask = new GenTask(pos, new WeakReference<>(fullDataSource));
-		worldGenQueue.submitGenTask(fullDataSource.getSectionPos().getSectionBBoxPos(), fullDataSource.getDataDetailLevel(), genTask)
+		//LOGGER.info("["+ungeneratedPosList.size()+"] missing sub positions for pos: ["+pos+"]. Number of gen requests queued: ["+this.incompleteSourceGenRequests.size()+"].");
+		
+		List<CompletableFuture<WorldGenResult>> futureList = new ArrayList<>();
+		for (DhSectionPos ungenChildPos : ungeneratedPosList)
+		{
+			// don't queue the same section twice
+			if (this.incompleteSourceGenRequests.contains(ungenChildPos))
+			{
+				continue;
+			}
+			this.incompleteSourceGenRequests.add(ungenChildPos);
+			
+			
+			// FIXME this can cause duplicate terrain generation requests
+			GenTask genTask = new GenTask(ungenChildPos, new WeakReference<>(fullDataSource));
+			CompletableFuture<WorldGenResult> future = worldGenQueue.submitGenTask(new DhLodPos(ungenChildPos), fullDataSource.getDataDetailLevel(), genTask)
 				.whenComplete((genTaskResult, ex) ->
 				{
-					incompleteSourceGenRequests.remove(pos);
-					//LOGGER.info("Partial generation completed for pos: ["+pos+"]. Remaining gen requests queued: ["+queuedGenRequests.size()+"].");
-
-					this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos);
+					//LOGGER.info("Partial generation completed for pos: ["+pos+"]. Remaining gen requests queued: ["+this.incompleteSourceGenRequests.size()+"].");
+					
+					this.onWorldGenTaskComplete(genTaskResult, ex, genTask, ungenChildPos);
+					this.fireOnGenPosSuccessListeners(pos);
+					this.incompleteSourceGenRequests.remove(ungenChildPos);
 				});
+			futureList.add(future);
+		}
+		
+		
+		if (futureList.size() != 0)
+		{
+			CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0]))
+				.whenComplete((genTaskResult, ex) ->
+				{
+					// parent pos has completed generation
+					this.fireOnGenPosSuccessListeners(pos);
+					this.incompleteSourceGenRequests.remove(pos);
+				});
+		}
 	}
 	
 	private void onWorldGenTaskComplete(WorldGenResult genTaskResult, Throwable exception, GenTask genTask, DhSectionPos pos)
@@ -231,33 +269,39 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 		else if (genTaskResult.success)
 		{
 			// generation completed, update the files and listener(s)
-			
-//			LOGGER.info("gen task completed for pos: ["+pos+"].");
-			this.files.get(genTask.pos).flushAndSaveAsync();
-			
-			// fire the event listeners 
-			for (IOnWorldGenCompleteListener listener : this.onWorldGenTaskCompleteListeners)
-			{
-				listener.onWorldGenTaskComplete(genTask.pos);
-			}
-			
-//			this.files.get(genTask.pos).metaData.dataVersion.incrementAndGet();
+			this.fireOnGenPosSuccessListeners(pos);
 			return;
 		}
 		else
 		{
 			// generation didn't complete
-			
-			// if the generation task was split up into smaller positions, wait for them to complete
-			for (CompletableFuture<WorldGenResult> siblingFuture : genTaskResult.childFutures)
-			{
-				siblingFuture.whenComplete((siblingGenTaskResult, siblingEx) -> this.onWorldGenTaskComplete(siblingGenTaskResult, siblingEx, genTask, pos));
-			}
+		}
+		
+		
+		// if the generation task was split up into smaller positions, add the on-complete event to them
+		for (CompletableFuture<WorldGenResult> siblingFuture : genTaskResult.childFutures)
+		{
+			siblingFuture.whenComplete((siblingGenTaskResult, siblingEx) -> this.onWorldGenTaskComplete(siblingGenTaskResult, siblingEx, genTask, pos));
 		}
 		
 		genTask.releaseStrongReference();
 	}
 	
+	private void fireOnGenPosSuccessListeners(DhSectionPos pos)
+	{
+		//LOGGER.info("gen task completed for pos: ["+pos+"].");
+		
+		this.fileBySectionPos.get(pos).flushAndSaveAsync()
+			.whenComplete((voidObj, ex) ->
+			{
+				// fire the event listeners 
+				for (IOnWorldGenCompleteListener listener : this.onWorldGenTaskCompleteListeners)
+				{
+					listener.onWorldGenTaskComplete(pos);
+				}
+			});
+	}
+	
 	
 	
 	//================//
@@ -292,11 +336,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 			if (this.loadedTargetFullDataSource == null)
 			{
 				this.loadedTargetFullDataSource = this.targetFullDataSourceRef.get();
-				if (this.loadedTargetFullDataSource == null)
-				{
-					return null;
-				}
 			}
+			if (this.loadedTargetFullDataSource == null)
+			{
+				return null;
+			}
+			
 			
 			return (chunkSizedFullDataSource) ->
 			{
@@ -311,8 +356,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 		
 	}
 	
-	/** 
-	 * used by external event listeners <br> 
+	/**
+	 * used by external event listeners <br>
 	 * TODO may or may not be best to have this in a separate file
 	 */
 	@FunctionalInterface

core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java

@@ -30,7 +30,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
 	 * When clearing, don't set to null, instead create a SoftReference containing null. 
 	 * This will make null checks simpler.
 	 */
-	private SoftReference<ColumnRenderSource> cachedRenderDataSource = new SoftReference<>(null);
+	private SoftReference<ColumnRenderSource> cachedRenderDataSourceRef = new SoftReference<>(null);
 	
 	private final RenderSourceFileHandler fileHandler;
 	private boolean doesFileExist;
@@ -110,7 +110,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
 	private CompletableFuture<ColumnRenderSource> getCachedDataSourceAsync()
 	{
 		// attempt to get the cached data source
-		ColumnRenderSource cachedRenderDataSource = this.cachedRenderDataSource.get();
+		ColumnRenderSource cachedRenderDataSource = this.cachedRenderDataSourceRef.get();
 		if (cachedRenderDataSource != null)
 		{
 			return this.fileHandler.onReadRenderSourceLoadedFromCacheAsync(this, cachedRenderDataSource)
@@ -127,7 +127,6 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
 	
 	public CompletableFuture<ColumnRenderSource> loadOrGetCachedDataSourceAsync(Executor fileReaderThreads, IDhLevel level)
 	{
-		
 		CompletableFuture<ColumnRenderSource> getCachedFuture = this.getCachedDataSourceAsync();
 		if (getCachedFuture != null)
 		{
@@ -157,12 +156,12 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
 					{
 						LOGGER.error("Uncaught error on creation {}: ", this.file, ex);
 						loadRenderSourceFuture.complete(null);
-						this.cachedRenderDataSource = new SoftReference<>(null);
+						this.cachedRenderDataSourceRef = new SoftReference<>(null);
 					}
 					else
 					{
 						loadRenderSourceFuture.complete(renderSource);
-						this.cachedRenderDataSource = new SoftReference<>(renderSource);
+						this.cachedRenderDataSourceRef = new SoftReference<>(renderSource);
 					}
 				});
 		}
@@ -196,12 +195,12 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
 					{
 						LOGGER.error("Error loading file {}: ", this.file, ex);
 						loadRenderSourceFuture.complete(null);
-						this.cachedRenderDataSource = new SoftReference<>(null);
+						this.cachedRenderDataSourceRef = new SoftReference<>(null);
 					}
 					else
 					{
 						loadRenderSourceFuture.complete(renderSource);
-						this.cachedRenderDataSource = new SoftReference<>(renderSource);
+						this.cachedRenderDataSourceRef = new SoftReference<>(renderSource);
 					}
 				});
 		}

core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java

@@ -6,6 +6,7 @@ import com.seibel.lod.core.config.Config;
 import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
 import com.seibel.lod.core.dataObjects.transformers.LodDataBuilder;
 import com.seibel.lod.core.dependencyInjection.SingletonInjector;
+import com.seibel.lod.core.file.fullDatafile.GeneratedFullDataFileHandler;
 import com.seibel.lod.core.generation.tasks.*;
 import com.seibel.lod.core.pos.*;
 import com.seibel.lod.core.util.ThreadUtil;
@@ -19,6 +20,7 @@ import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper;
 import org.apache.logging.log4j.Logger;
 
 import java.io.Closeable;
+import java.lang.ref.WeakReference;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.function.Consumer;
@@ -35,11 +37,13 @@ public class WorldGenerationQueue implements Closeable
 	private final ConcurrentHashMap<DhLodPos, InProgressWorldGenTaskGroup> inProgressGenTasksByLodPos = new ConcurrentHashMap<>();
 	
 	// granularity is the detail level for batching world generator requests together
-	private final byte maxGranularity;
-	private final byte minGranularity;
+	public final byte maxGranularity;
+	public final byte minGranularity;
 	
-	private final byte maxDataDetail;
-	private final byte minDataDetail;
+	/** largest numerical detail level allowed */
+	 final byte largestDataDetail;
+	/** lowest numerical detail level allowed */
+	public final byte smallestDataDetail;
 	
 	/** If not null this generator is in the process of shutting down */
 	private volatile CompletableFuture<Void> generatorClosingFuture = null;
@@ -54,24 +58,26 @@ public class WorldGenerationQueue implements Closeable
 	private DhBlockPos2D generationTargetPos = DhBlockPos2D.ZERO;
 	/** can be used for debugging how many tasks are currently in the queue */
 	private int numberOfTasksQueued = 0;
-	/** 
-	 * Settings this to true will cause the system to queue the first generation request it can find, 
-	 * this improves generation thread feeding efficiency, but can potentially cause generation requests to queue out of order (IE they may not be closest to farthest). <br><br>
-	 * 
-	 * Setting this to false will cause the system to queue the closest generation request it can find. <br>
-	 * This will reduce generation queuing efficiency.
-	 */
-	private boolean queueFirstGenerationRequestFound = true; 
+	
+	// debug variables to test for duplicate world generator requests //
+	/** limits how many of the previous world gen requests we should track */
+	private static final int MAX_ALREADY_GENERATED_COUNT = 400;
+	private final HashSet<DhLodPos> alreadyGeneratedPosHashSet = new HashSet<>(MAX_ALREADY_GENERATED_COUNT);
+	private final Queue<DhLodPos> alreadyGeneratedPosQueue = new LinkedList<>();
 	
 	
 	
+	//==============//
+	// constructors //
+	//==============//
+	
 	public WorldGenerationQueue(IDhApiWorldGenerator generator)
 	{
 		this.generator = generator;
 		this.maxGranularity = generator.getMaxGenerationGranularity();
 		this.minGranularity = generator.getMinGenerationGranularity();
-		this.maxDataDetail = generator.getLargestDataDetailLevel();
-		this.minDataDetail = generator.getSmallestDataDetailLevel();
+		this.largestDataDetail = generator.getLargestDataDetailLevel();
+		this.smallestDataDetail = generator.getSmallestDataDetailLevel();
 		
 		int treeWidth = Config.Client.Graphics.Quality.lodChunkRenderDistance.get() * LodUtil.CHUNK_WIDTH * 2; // TODO the *2 is to allow for generation edge cases, and should probably be removed at some point
 		byte treeMinDetailLevel = LodUtil.BLOCK_DETAIL_LEVEL; // the tree shouldn't need to go this low, but just in case
@@ -105,13 +111,13 @@ public class WorldGenerationQueue implements Closeable
 		
 		
 		// make sure the generator can provide the requested position
-		if (requiredDataDetail < this.minDataDetail)
+		if (requiredDataDetail < this.smallestDataDetail)
 		{
 			throw new UnsupportedOperationException("Current generator does not meet requiredDataDetail level");
 		}
-		if (requiredDataDetail > this.maxDataDetail)
+		if (requiredDataDetail > this.largestDataDetail)
 		{
-			requiredDataDetail = this.maxDataDetail;
+			requiredDataDetail = this.largestDataDetail;
 		}
 		
 		// TODO what does this assert mean?
@@ -358,9 +364,36 @@ public class WorldGenerationQueue implements Closeable
 		DhLodPos taskPos = inProgressTaskGroup.group.pos;
 		byte granularity = (byte) (taskPos.detailLevel - taskDetailLevel);
 		LodUtil.assertTrue(granularity >= this.minGranularity && granularity <= this.maxGranularity);
-		LodUtil.assertTrue(taskDetailLevel >= this.minDataDetail && taskDetailLevel <= this.maxDataDetail);
+		LodUtil.assertTrue(taskDetailLevel >= this.smallestDataDetail && taskDetailLevel <= this.largestDataDetail);
 		
 		DhChunkPos chunkPosMin = new DhChunkPos(taskPos.getCornerBlockPos());
+		
+		
+		// Could be enabled (change the if-false to true) to test for duplicate world generator requests
+		if (false)
+		{
+			if (this.alreadyGeneratedPosHashSet.contains(inProgressTaskGroup.group.pos))
+			{
+				// temporary solution to prevent generating the same section multiple times
+				LOGGER.warn("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping...");
+				
+				// FIXME this prevents sections from generating, or from being updated in the LodQuadTree, James isn't sure which issue it is
+				inProgressTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateFail()));
+				return;
+			}
+			
+			this.alreadyGeneratedPosHashSet.add(inProgressTaskGroup.group.pos);
+			this.alreadyGeneratedPosQueue.add(inProgressTaskGroup.group.pos);
+			
+			
+			// remove extra tracked positions
+			while (this.alreadyGeneratedPosQueue.size() > MAX_ALREADY_GENERATED_COUNT)
+			{
+				DhLodPos posToRemove = this.alreadyGeneratedPosQueue.poll();
+				this.alreadyGeneratedPosHashSet.remove(posToRemove);
+			}
+		}
+		
 		//LOGGER.info("Generating section "+taskPos+" with granularity "+granularity+" at "+chunkPosMin);
 		
 		this.numberOfTasksQueued++;
@@ -371,7 +404,8 @@ public class WorldGenerationQueue implements Closeable
 			if (exception != null)
 			{
 				// don't log the shutdown exceptions
-				if (!UncheckedInterruptedException.isThrowableInterruption(exception) && !(exception instanceof CancellationException || exception.getCause() instanceof CancellationException))
+				if (!UncheckedInterruptedException.isThrowableInterruption(exception)
+					&& !(exception instanceof CancellationException || exception.getCause() instanceof CancellationException))
 				{
 					LOGGER.error("Error generating data for section "+taskPos, exception);
 				}