Optimization on load time, fix gl bug, and improve transparency

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java

@@ -8,15 +8,18 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIn
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
+import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhLodPos;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
+import com.seibel.distanthorizons.core.util.FileScanUtil;
 import com.seibel.distanthorizons.core.util.FileUtil;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.util.ThreadUtil;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.io.IOException;
@@ -26,43 +29,74 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
+import static com.seibel.distanthorizons.core.util.FileScanUtil.LOD_FILE_POSTFIX;
+import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX;
+
 public class FullDataFileHandler implements IFullDataSourceProvider
 {
+	public static final boolean USE_LAZY_LOADING = true;
+
     private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	
 	protected static ExecutorService fileHandlerThreadPool;
 	protected static ConfigChangeListener<Integer> configListener;
-	
-	protected final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> fileBySectionPos = new ConcurrentHashMap<>();
+
+	private final ConcurrentHashMap<DhSectionPos, File> unloadedFiles = new ConcurrentHashMap<>();
+	private final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> fileBySectionPos = new ConcurrentHashMap<>();
+	public void ForEachFile(Consumer<FullDataMetaFile> consumer) {
+		fileBySectionPos.values().forEach(consumer);
+	}
+
 	protected final IDhLevel level;
 	protected final File saveDir;
 	protected final AtomicInteger topDetailLevel = new AtomicInteger(-1);
 	protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET;
 	
-	
-	
-    public FullDataFileHandler(IDhLevel level, File saveRootDir)
+    public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure)
 	{
 		this.level = level;
-        this.saveDir = saveRootDir;
+        this.saveDir = saveStructure.getFullDataFolder(level.getLevelWrapper());
 		if (!this.saveDir.exists() && !this.saveDir.mkdirs())
 		{
 			LOGGER.warn("Unable to create full data folder, file saving may fail.");
 		}
+		FileScanUtil.scanFiles(saveStructure, level.getLevelWrapper(), this, null);
     }
-	
-	
-	
-    /**
-    * Caller must ensure that this method is called only once,
-    *  and that the {@link FullDataFileHandler} is not used before this method is called.
-     */
-    @Override
-    public void addScannedFile(Collection<File> detectedFiles)
+
+	/**
+	 * Caller must ensure that this method is called only once,
+	 *  and that the {@link FullDataFileHandler} is not used before this method is called.
+	 */
+	@Override
+	public void addScannedFile(Collection<File> detectedFiles)
+	{
+		if (USE_LAZY_LOADING) {
+			lazyAddScannedFile(detectedFiles);
+		}
+		else {
+			immediateAddScannedFile(detectedFiles);
+		}
+	}
+
+	private void lazyAddScannedFile(Collection<File> detectedFiles) {
+		for (File file : detectedFiles) {
+			try {
+				DhSectionPos pos = decodePositionByFile(file);
+				if (pos != null) {
+					unloadedFiles.put(pos, file);
+					this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
+				}
+			}
+			catch (Exception e) {
+				LOGGER.error("Failed to read data meta file at " + file + ": ", e);
+				FileUtil.renameCorruptedFile(file);
+			}
+		}
+	}
+
+	private void immediateAddScannedFile(Collection<File> detectedFiles)
 	{
         HashMultimap<DhSectionPos, FullDataMetaFile> filesByPos = HashMultimap.create();
-        LOGGER.info("Detected {} valid files in {}", detectedFiles.size(), this.saveDir);
-
         { // Sort files by pos.
             for (File file : detectedFiles)
 			{
@@ -138,29 +172,52 @@ public class FullDataFileHandler implements IFullDataSourceProvider
         }
     }
 	
-    protected FullDataMetaFile getOrMakeFile(DhSectionPos pos)
+    protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile)
 	{
         FullDataMetaFile metaFile = this.fileBySectionPos.get(pos);
-        if (metaFile == null)
-		{
-            FullDataMetaFile newMetaFile;
-            try
-			{
-                newMetaFile = new FullDataMetaFile(this, this.level, pos);
-            }
-			catch (IOException e)
-			{
-                LOGGER.error("IOException on creating new data file at {}", pos, e);
-                return null;
-            }
-			
-            metaFile = this.fileBySectionPos.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
-            if (metaFile == null) 
-			{
-				metaFile = newMetaFile;
+		if (metaFile != null) return metaFile;
+
+		File fileToLoad = unloadedFiles.get(pos);
+		// File does exist, but not loaded yet.
+		if (fileToLoad != null) {
+			synchronized (this) {
+				// Double check locking for loading file, as loading file means also loading the metadata, which
+				// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
+				// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
+				metaFile = this.fileBySectionPos.get(pos);
+				if (metaFile != null) return metaFile; // someone else loaded it already.
+				try {
+					metaFile = new FullDataMetaFile(this, this.level, fileToLoad);
+					this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
+					this.fileBySectionPos.put(pos, metaFile);
+					return metaFile;
+				}
+				catch (IOException e) {
+					LOGGER.error("Failed to read data meta file at " + fileToLoad + ": ", e);
+					FileUtil.renameCorruptedFile(fileToLoad);
+				}
+				finally {
+					unloadedFiles.remove(pos);
+				}
 			}
-        }
-        return metaFile;
+		}
+		if (!allowCreateFile) return null;
+		// File does not exist, create it.
+		// In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS
+		// to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects.
+		try
+		{
+			metaFile = new FullDataMetaFile(this, this.level, pos);
+		}
+		catch (IOException e)
+		{
+			LOGGER.error("IOException on creating new data file at {}", pos, e);
+			return null;
+		}
+		// This is a CAS with expected null value.
+		this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
+		FullDataMetaFile metaFileCas = this.fileBySectionPos.putIfAbsent(pos, metaFile);
+		return metaFileCas == null ? metaFile : metaFileCas;
     }
 	
 	/**
@@ -247,14 +304,16 @@ public class FullDataFileHandler implements IFullDataSourceProvider
 	/**
 	 * Returns the {@link IFullDataSource} for the given section position. <Br>
 	 * The returned data source may be null. <Br> <Br>
-	 * 
+	 *
+	 * For now, if result is null, it prob means error has occurred when loading or creating the file object. <Br> <Br>
+	 *
 	 * This call is concurrent. I.e. it supports being called by multiple threads at the same time.
 	 */
 	@Override
     public CompletableFuture<IFullDataSource> read(DhSectionPos pos)
 	{
 		this.topDetailLevel.updateAndGet(intVal -> Math.max(intVal, pos.sectionDetailLevel));
-        FullDataMetaFile metaFile = this.getOrMakeFile(pos);
+        FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true);
         if (metaFile == null)
 		{
 			return CompletableFuture.completedFuture(null);
@@ -386,7 +445,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
 	protected void makeFiles(ArrayList<DhSectionPos> posList, ArrayList<FullDataMetaFile> output) {
 		for (DhSectionPos missingPos : posList)
 		{
-			FullDataMetaFile newFile = this.getOrMakeFile(missingPos);
+			FullDataMetaFile newFile = this.getLoadOrMakeFile(missingPos, true);
 			if (newFile != null)
 			{
 				output.add(newFile);
@@ -427,7 +486,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
 		// remove the FullDataMetaFile since the old one was corrupted
 		this.fileBySectionPos.remove(pos);
 		// create a new FullDataMetaFile to write new data to
-		return this.getOrMakeFile(pos);
+		return this.getLoadOrMakeFile(pos, true);
 	}
 	
 	@Override
@@ -455,8 +514,16 @@ public class FullDataFileHandler implements IFullDataSourceProvider
     }
 	
     @Override
-    public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + ".lod"); }
-	
+    public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + LOD_FILE_POSTFIX); }
+
+	@Nullable
+	public DhSectionPos decodePositionByFile(File file)
+	{
+		String fileName = file.getName();
+		if (!fileName.endsWith(LOD_FILE_POSTFIX)) return null;
+		fileName = fileName.substring(0, fileName.length() - 4);
+		return DhSectionPos.deserialize(fileName);
+	}
 	
 	
 	//==========================//

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java

@@ -4,7 +4,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedF
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
-import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
+import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.generation.WorldGenerationQueue;
 import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
 import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
@@ -13,8 +13,6 @@ import com.seibel.distanthorizons.core.level.IDhServerLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhLodPos;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
-import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource;
-import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import org.apache.logging.log4j.Logger;
 
@@ -26,8 +24,6 @@ import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public class GeneratedFullDataFileHandler extends FullDataFileHandler
 {
@@ -40,7 +36,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 	// Use to hold onto incomplete data sources that are waiting for generation, so that they don't get GC'd before they are generated
 	private final ConcurrentHashMap<DhSectionPos, IIncompleteFullDataSource> incompleteDataSources = new ConcurrentHashMap<>();
 	
-	public GeneratedFullDataFileHandler(IDhServerLevel level, File saveRootDir) { super(level, saveRootDir); }
+	public GeneratedFullDataFileHandler(IDhServerLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); }
 	
 	
 	
@@ -69,17 +65,14 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 		boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue);
 		LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!");
 		LOGGER.info("Set world gen queue for level {} to start.", this.level);
-		for (FullDataMetaFile metaFile : this.fileBySectionPos.values())
-		{
+		this.ForEachFile(metaFile -> {
 			IFullDataSource data = metaFile.getCachedDataSourceNowOrNull();
-			if (data instanceof CompleteFullDataSource) {
-				continue;
-			}
+			if (data instanceof CompleteFullDataSource) return;
 			metaFile.genQueueChecked = false; // unset it so it can be checked again
 			if (data != null) {
 				metaFile.markNeedUpdate();
 			}
-		}
+		});
 		flushAndSave(); // Trigger an update to the meta files
 	}
 	
@@ -130,7 +123,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
 			if (targetDataDetailLevel > maxSectDataDetailLevel) {
 				ArrayList<FullDataMetaFile> existingFiles = new ArrayList<>();
 				byte sectDetailLevel = (byte) (DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + maxSectDataDetailLevel);
-				pos.forEachChildAtLevel(sectDetailLevel, p -> existingFiles.add(getOrMakeFile(p)));
+				pos.forEachChildAtLevel(sectDetailLevel, p -> existingFiles.add(getLoadOrMakeFile(p, true)));
 				return sampleFromFiles(dataSource, existingFiles).thenApply(this::tryPromoteDataSource)
 						.exceptionally((e) ->
 						{

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java

@@ -1,11 +1,12 @@
 package com.seibel.distanthorizons.core.file.fullDatafile;
 
+import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 
 import java.io.File;
 
 public class RemoteFullDataFileHandler extends FullDataFileHandler
 {
-    public RemoteFullDataFileHandler(IDhLevel level, File saveRootDir) { super(level, saveRootDir); }
+    public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); }
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java

@@ -1,8 +1,11 @@
 package com.seibel.distanthorizons.core.file.renderfile;
 
 import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
 import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
+import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
+import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhLodPos;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
@@ -11,30 +14,34 @@ import com.seibel.distanthorizons.core.dataObjects.transformers.DataRenderTransf
 import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
 import com.seibel.distanthorizons.core.level.IDhClientLevel;
 import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
+import com.seibel.distanthorizons.core.util.FileScanUtil;
 import com.seibel.distanthorizons.core.util.FileUtil;
 import com.seibel.distanthorizons.core.util.ThreadUtil;
 import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException;
 import com.seibel.distanthorizons.core.config.Config;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 
 import java.awt.*;
 import java.io.File;
 import java.io.IOException;
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
+import java.util.List;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX;
+
 public class RenderSourceFileHandler implements ILodRenderSourceProvider
 {
-	public static final String RENDER_FILE_EXTENSION = ".rlod";
+	public static final boolean USE_LAZY_LOADING = true;
 	public static final long RENDER_SOURCE_TYPE_ID = ColumnRenderSource.TYPE_ID;
 	
     private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	
 	private final ExecutorService fileHandlerThreadPool = ThreadUtil.makeSingleThreadPool("Render Source File Handler");
+
+	private final ConcurrentHashMap<DhSectionPos, File> unloadedFiles = new ConcurrentHashMap<>();
 	private final ConcurrentHashMap<DhSectionPos, RenderMetaDataFile> filesBySectionPos = new ConcurrentHashMap<>();
 	
 	private final IDhClientLevel level;
@@ -42,21 +49,18 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 	/** This is the lowest (highest numeric) detail level that this {@link RenderSourceFileHandler} is keeping track of. */
 	AtomicInteger topDetailLevel = new AtomicInteger(6);
 	private final IFullDataSourceProvider fullDataSourceProvider;
-	
-	private final ConcurrentHashMap<DhSectionPos, Object> cacheUpdateLockBySectionPos = new ConcurrentHashMap<>();
-	
-	
-	
-	
-	public RenderSourceFileHandler(IFullDataSourceProvider sourceProvider, IDhClientLevel level, File saveRootDir)
+
+
+	public RenderSourceFileHandler(IFullDataSourceProvider sourceProvider, IDhClientLevel level, AbstractSaveStructure saveStructure)
 	{
         this.fullDataSourceProvider = sourceProvider;
         this.level = level;
-        this.saveDir = saveRootDir;
+        this.saveDir = saveStructure.getRenderCacheFolder(level.getLevelWrapper());
 		if (!this.saveDir.exists() && !this.saveDir.mkdirs())
 		{
 			LOGGER.warn("Unable to create render data folder, file saving may fail.");
 		}
+		FileScanUtil.scanFiles(saveStructure, level.getLevelWrapper(), null, this);
     }
 	
 	
@@ -70,7 +74,33 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 	 * and that the given files are not used before this method is called.
 	 */
 	@Override
-	public void addScannedFile(Collection<File> newRenderFiles)
+	public void addScannedFile(Collection<File> detectedFiles)
+	{
+		if (USE_LAZY_LOADING) {
+			lazyAddScannedFile(detectedFiles);
+		}
+		else {
+			immediateAddScannedFile(detectedFiles);
+		}
+	}
+
+	private void lazyAddScannedFile(Collection<File> detectedFiles) {
+		for (File file : detectedFiles) {
+			try {
+				DhSectionPos pos = decodePositionByFile(file);
+				if (pos != null) {
+					unloadedFiles.put(pos, file);
+					this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
+				}
+			}
+			catch (Exception e) {
+				LOGGER.error("Failed to read data meta file at " + file + ": ", e);
+				FileUtil.renameCorruptedFile(file);
+			}
+		}
+	}
+
+	private void immediateAddScannedFile(Collection<File> newRenderFiles)
 	{
 		HashMultimap<DhSectionPos, RenderMetaDataFile> filesByPos = HashMultimap.create();
 		
@@ -89,8 +119,6 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 			}
 		}
 		
-		
-		
 		// Warn for multiple files with the same pos, and then select the one with the latest timestamp.
 		for (DhSectionPos pos : filesByPos.keySet())
 		{
@@ -148,19 +176,61 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 			{
 				fileToUse = metaFiles.iterator().next();
 			}
-			
-			
 			// Add this file to the list of files.
 			this.filesBySectionPos.put(pos, fileToUse);
-			
 			// increase the lowest detail level if a new lower detail file is found
-			if (this.topDetailLevel.get() < pos.sectionDetailLevel)
-			{
-				this.topDetailLevel.set(pos.sectionDetailLevel);
-			}
+			this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
 		}
 	}
-	
+
+	protected RenderMetaDataFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile)
+	{
+		RenderMetaDataFile metaFile = this.filesBySectionPos.get(pos);
+		if (metaFile != null) return metaFile;
+
+		File fileToLoad = unloadedFiles.get(pos);
+		// File does exist, but not loaded yet.
+		if (fileToLoad != null) {
+			synchronized (this) {
+				// Double check locking for loading file, as loading file means also loading the metadata, which
+				// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
+				// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
+				metaFile = this.filesBySectionPos.get(pos);
+				if (metaFile != null) return metaFile; // someone else loaded it already.
+				try {
+					metaFile = RenderMetaDataFile.createFromExistingFile(this, fileToLoad);
+					this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
+					this.filesBySectionPos.put(pos, metaFile);
+					return metaFile;
+				}
+				catch (IOException e) {
+					LOGGER.error("Failed to read render meta file at " + fileToLoad + ": ", e);
+					FileUtil.renameCorruptedFile(fileToLoad);
+				}
+				finally {
+					unloadedFiles.remove(pos);
+				}
+			}
+		}
+		if (!allowCreateFile) return null;
+		// File does not exist, create it.
+		// In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS
+		// to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects.
+		try
+		{
+			metaFile = RenderMetaDataFile.createNewFileForPos(this, pos);
+		}
+		catch (IOException e)
+		{
+			LOGGER.error("IOException on creating new data file at {}", pos, e);
+			return null;
+		}
+		this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
+		// This is a CAS with expected null value.
+		RenderMetaDataFile metaFileCas = this.filesBySectionPos.putIfAbsent(pos, metaFile);
+		return metaFileCas == null ? metaFile : metaFileCas;
+	}
+
 	/** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */
     @Override
     public CompletableFuture<ColumnRenderSource> readAsync(DhSectionPos pos)
@@ -171,45 +241,10 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 			return CompletableFuture.completedFuture(null);
 		}
 		
-		
-		
-        RenderMetaDataFile metaFile = this.filesBySectionPos.get(pos);
-		if (metaFile == null)
-		{
-			RenderMetaDataFile newMetaFile;
-			try
-			{
-				File renderMetaFile = this.computeRenderFilePath(pos);
-				boolean renderFileExists = renderMetaFile.exists();
-				
-				if (renderFileExists)
-				{
-					newMetaFile = RenderMetaDataFile.createFromExistingFile(this, renderMetaFile);
-				}
-				else
-				{
-					newMetaFile = RenderMetaDataFile.createNewFileForPos(this, pos);
-				}
-			}
-			catch (IOException e)
-			{
-				LOGGER.error("IOException on creating new render file at "+pos, e);
-				return null;
-			}
-			
-			
-			metaFile = this.filesBySectionPos.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
-			if (metaFile == null)
-			{
-				metaFile = newMetaFile;
-			}
-			
-			// increase the lowest detail level if a new lower detail file was added
-			if (this.topDetailLevel.get() < pos.sectionDetailLevel)
-			{
-				this.topDetailLevel.set(pos.sectionDetailLevel);
-			}
-		}
+        RenderMetaDataFile metaFile = this.getLoadOrMakeFile(pos, true);
+
+		// On error, (when it returns null,) return an empty render source
+		if (metaFile == null) return CompletableFuture.completedFuture(ColumnRenderSource.createEmptyRenderSource(pos));
 		
         return metaFile.loadOrGetCachedDataSourceAsync(this.fileHandlerThreadPool, this.level).handle(
 			(renderSource, exception) ->
@@ -258,7 +293,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 		for (int ox = 0; ox < width; ox++) {
 			for (int oz = 0; oz < width; oz++) {
 				DhSectionPos sectPos = new DhSectionPos(sectionDetailLevel, sectPosMin.x + ox, sectPosMin.z + oz);
-				RenderMetaDataFile metaFile = this.filesBySectionPos.get(sectPos);
+				RenderMetaDataFile metaFile = this.filesBySectionPos.get(sectPos); // bypass the getLoadOrMakeFile(), as we only want in-cache files.
 				if (metaFile != null)
 				{
 					metaFile.updateChunkIfSourceExists(chunk, this.level);
@@ -431,9 +466,12 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 				}
 			}
 		}
-		
+		else {
+			renderFiles = new File[0];
+		}
 		// clear the cached files
 		this.filesBySectionPos.clear();
+		addScannedFile(ImmutableList.copyOf(renderFiles));
 	}
 	
 	
@@ -442,7 +480,15 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
 	// helper methods //
 	//================//
 	
-	public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RENDER_FILE_EXTENSION);}
-	
+	public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RENDER_FILE_POSTFIX);}
+
+	@Nullable
+	public DhSectionPos decodePositionByFile(File file)
+	{
+		String fileName = file.getName();
+		if (!fileName.endsWith(RENDER_FILE_POSTFIX)) return null;
+		fileName = fileName.substring(0, fileName.length() - RENDER_FILE_POSTFIX.length());
+		return DhSectionPos.deserialize(fileName);
+	}
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java

@@ -221,7 +221,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable
 					break;
 				}
 				IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), clientLevelWrapper);
-				IFullDataSourceProvider fileHandler = new FullDataFileHandler(tempLevel, testLevelFolder);
+				IFullDataSourceProvider fileHandler = new FullDataFileHandler(tempLevel, tempLevel.getSaveStructure());
 				CompletableFuture<IFullDataSource> testDataSource = fileHandler.read(new DhSectionPos(playerChunkPos));
 				IFullDataSource lodDataSource = testDataSource.get(); 
 				

core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java

@@ -84,8 +84,6 @@ public class ClientLevelModule {
         if (isBuffersDirty) {
             clientRenderState.renderer.bufferHandler.MarkAllBuffersDirty();
         }
-
-        clientRenderState.renderer.bufferHandler.updateQuadTreeRenderSources();
     }
 
 
@@ -256,13 +254,12 @@ public class ClientLevelModule {
                 AbstractSaveStructure saveStructure)
         {
             this.levelWrapper = dhClientLevel.getLevelWrapper();
-            this.renderSourceFileHandler = new RenderSourceFileHandler(fullDataSourceProvider, dhClientLevel, saveStructure.getRenderCacheFolder(this.levelWrapper));
+            this.renderSourceFileHandler = new RenderSourceFileHandler(fullDataSourceProvider, dhClientLevel, saveStructure);
 
             this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistance.get() * LodUtil.CHUNK_WIDTH,
                     MC_CLIENT.getPlayerBlockPos().x, MC_CLIENT.getPlayerBlockPos().z, this.renderSourceFileHandler);
 
             RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree);
-            FileScanUtil.scanFiles(saveStructure, this.levelWrapper, fullDataSourceProvider, this.renderSourceFileHandler);
             this.renderer = new LodRenderer(renderBufferHandler);
         }
 

core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java

@@ -32,13 +32,9 @@ public class DhClientLevel extends DhLevel implements IDhClientLevel
 	
 	public DhClientLevel(AbstractSaveStructure saveStructure, IClientLevelWrapper clientLevelWrapper)
 	{
-		if (saveStructure.getFullDataFolder(clientLevelWrapper).mkdirs())
-		{
-			LOGGER.warn("unable to create data folder.");
-		}
 		this.levelWrapper = clientLevelWrapper;
 		this.saveStructure = saveStructure;
-		dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure.getFullDataFolder(levelWrapper));
+		dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure);
 		clientside = new ClientLevelModule(this);
 		clientside.startRenderer();
 		LOGGER.info("Started DHLevel for "+this.levelWrapper+" with saves at "+this.saveStructure);

core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java

@@ -31,6 +31,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS
 	public final ServerLevelModule serverside;
 	public final ClientLevelModule clientside;
 
+	private final IServerLevelWrapper serverLevelWrapper;
 	public IClientLevelWrapper clientLevelWrapper;
 	
 	public DhClientServerLevel(AbstractSaveStructure saveStructure, IServerLevelWrapper serverLevelWrapper)
@@ -39,6 +40,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS
 		{
 			LOGGER.warn("unable to create data folder.");
 		}
+		this.serverLevelWrapper = serverLevelWrapper;
 		serverside = new ServerLevelModule(this, serverLevelWrapper, saveStructure);
 		clientside = new ClientLevelModule(this);
 		LOGGER.info("Started "+DhClientServerLevel.class.getSimpleName()+" for "+ serverLevelWrapper +" with saves at "+saveStructure);
@@ -135,7 +137,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS
 	}
 
 	@Override
-	public IServerLevelWrapper getServerLevelWrapper() { return serverside.levelWrapper; }
+	public IServerLevelWrapper getServerLevelWrapper() { return serverLevelWrapper; }
 	@Override
 	public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); }
 

core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java

@@ -18,6 +18,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
 {
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	public final ServerLevelModule serverside;
+	private final IServerLevelWrapper serverLevelWrapper;
 
 	public DhServerLevel(AbstractSaveStructure saveStructure, IServerLevelWrapper serverLevelWrapper)
 	{
@@ -25,6 +26,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
 		{
 			LOGGER.warn("unable to create data folder.");
 		}
+		this.serverLevelWrapper = serverLevelWrapper;
 		serverside = new ServerLevelModule(this, serverLevelWrapper, saveStructure);
 		LOGGER.info("Started DHLevel for {} with saves at {}", serverLevelWrapper, saveStructure);
 	}
@@ -83,7 +85,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
 	}
 	
 	@Override
-	public IServerLevelWrapper getServerLevelWrapper() { return serverside.levelWrapper; }
+	public IServerLevelWrapper getServerLevelWrapper() { return serverLevelWrapper; }
 	
 	@Override
 	public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); }

core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java

@@ -69,8 +69,7 @@ public class ServerLevelModule {
         this.parent = parent;
         this.levelWrapper = levelWrapper;
         this.saveStructure = saveStructure;
-        this.dataFileHandler = new GeneratedFullDataFileHandler(parent, saveStructure.getFullDataFolder(levelWrapper));
-        FileScanUtil.scanFiles(saveStructure, this.levelWrapper, this.dataFileHandler, null);
+        this.dataFileHandler = new GeneratedFullDataFileHandler(parent, saveStructure);
         this.worldGeneratorEnabledConfig = new AppliedConfigState<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration);
     }
 

core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java

@@ -3,6 +3,7 @@ package com.seibel.distanthorizons.core.pos;
 import com.seibel.distanthorizons.core.enums.ELodDirection;
 import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
 import com.seibel.distanthorizons.core.util.LodUtil;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.function.Consumer;
 
@@ -73,9 +74,8 @@ public class DhSectionPos
 		this.sectionX = dhLodPos.x;
 		this.sectionZ = dhLodPos.z;
 	}
-	
-	
-	
+
+
 	/** Returns the center for the highest detail level (0) */
 	public DhLodPos getCenter() { return this.getCenter((byte) 0); } // TODO why does this use detail level 0 instead of this object's detail level?
 	public DhLodPos getCenter(byte returnDetailLevel)
@@ -211,8 +211,15 @@ public class DhSectionPos
 	
 	/** Serialize() is different from toString() as it must NEVER be changed, and should be in a short format */
 	public String serialize() { return "[" + this.sectionDetailLevel + ',' + this.sectionX + ',' + this.sectionZ + ']'; }
-	
-	
+
+	@Nullable
+	public static DhSectionPos deserialize(String value) {
+		if (value.charAt(0) != '[' || value.charAt(value.length() - 1) != ']') return null;
+		String[] split = value.substring(1, value.length() - 1).split(",");
+		if (split.length != 3) return null;
+		return new DhSectionPos(Byte.parseByte(split[0]), Integer.parseInt(split[1]), Integer.parseInt(split[2]));
+
+	}
 	
 	@Override
 	public String toString() { return "{" + this.sectionDetailLevel + "*" + this.sectionX + "," + this.sectionZ + "}"; }

core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java

@@ -361,7 +361,6 @@ public class LodRenderSection implements IDebugRenderable
 	}
 
 	public void markBufferDirty() {
-		tellNeighborsUpdated();
 		lastSwapLocalVersion = -1;
 	}
 }

core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java

@@ -12,6 +12,8 @@ import org.apache.logging.log4j.Logger;
 
 import java.util.Comparator;
 import java.util.Iterator;
+import java.util.ListIterator;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /** 
  * This object tells the {@link LodRenderer} what buffers to render 
@@ -26,9 +28,10 @@ public class RenderBufferHandler
 	
 	// TODO: Make sorting go into the update loop instead of the render loop as it doesn't need to be done every frame
 	private SortedArraySet<LoadedRenderBuffer> loadedNearToFarBuffers = null;
-	
-	
-	
+
+	private final AtomicBoolean rebuildAllBuffers = new AtomicBoolean(false);
+
+
 	public RenderBufferHandler(LodQuadTree lodQuadTree) { this.lodQuadTree = lodQuadTree; }
 	
 	
@@ -40,7 +43,7 @@ public class RenderBufferHandler
 	 * TODO: This might get locked by update() causing move() call. Is there a way to avoid this?
 	 *       Maybe dupe the base list and use atomic swap on render? Or is this not worth it?
 	 */
-	public void buildRenderList(Vec3f lookForwardVector)
+	public void buildRenderListAndUpdateSections(Vec3f lookForwardVector)
 	{
 		ELodDirection[] axisDirections = new ELodDirection[3];
 		
@@ -95,12 +98,20 @@ public class RenderBufferHandler
 				axisDirections[2] = xDir;
 			}
 		}
+
+		Pos2D cPos = lodQuadTree.getCenterBlockPos().toPos2D();
 	
 		// Now that we have the axis directions, we can sort the render list
 		Comparator<LoadedRenderBuffer> farToNearComparator = (loadedBufferA, loadedBufferB) ->
 		{
 			Pos2D aPos = loadedBufferA.pos.getCenter().getCenterBlockPos().toPos2D();
 			Pos2D bPos = loadedBufferB.pos.getCenter().getCenterBlockPos().toPos2D();
+			if (true) {
+				int aManhattanDistance = aPos.manhattanDist(cPos);
+				int bManhattanDistance = bPos.manhattanDist(cPos);
+				return bManhattanDistance - aManhattanDistance;
+			}
+
 			for (ELodDirection axisDirection : axisDirections)
 			{
 				if (axisDirection.getAxis().isVertical())
@@ -135,7 +146,9 @@ public class RenderBufferHandler
 		
 		// Build the sorted list
 		this.loadedNearToFarBuffers = new SortedArraySet<>((a, b) -> -farToNearComparator.compare(a, b)); // TODO is the comparator named wrong?
-		
+
+		// Update the sections
+		boolean rebuildAllBuffers = this.rebuildAllBuffers.getAndSet(false);
 		Iterator<QuadNode<LodRenderSection>> nodeIterator = this.lodQuadTree.nodeIterator();
 		while (nodeIterator.hasNext())
 		{
@@ -143,15 +156,27 @@ public class RenderBufferHandler
 			
 			DhSectionPos sectionPos = node.sectionPos;
 			LodRenderSection renderSection = node.value;
-			
-			if (renderSection != null && renderSection.isRenderingEnabled())
-			{
-				AbstractRenderBuffer buffer = renderSection.activeRenderBufferRef.get();
-				if (buffer != null)
-				{
-					this.loadedNearToFarBuffers.add(new LoadedRenderBuffer(buffer, sectionPos));
+			try {
+
+				if (renderSection != null) {
+					if (rebuildAllBuffers) {
+						renderSection.markBufferDirty();
+					}
+					renderSection.tryBuildAndSwapBuffer();
+
+					if (renderSection.isRenderingEnabled()) {
+						AbstractRenderBuffer buffer = renderSection.activeRenderBufferRef.get();
+						if (buffer != null) {
+							this.loadedNearToFarBuffers.add(new LoadedRenderBuffer(buffer, sectionPos));
+						}
+					}
 				}
 			}
+			catch (Exception e)
+			{
+				LOGGER.error("Error updating QuadTree render source at "+renderSection.pos+".", e);
+				renderSection.markBufferDirty();
+			}
 		}
 	}
 	
@@ -163,41 +188,19 @@ public class RenderBufferHandler
     public void renderTransparent(LodRenderer renderContext)
 	{
 		//TODO: Directional culling
-		this.loadedNearToFarBuffers.forEach(loadedBuffer -> loadedBuffer.buffer.renderTransparent(renderContext));
+		ListIterator<LoadedRenderBuffer> iter = this.loadedNearToFarBuffers.listIterator(this.loadedNearToFarBuffers.size());
+		while (iter.hasPrevious())
+		{
+			LoadedRenderBuffer loadedBuffer = iter.previous();
+			loadedBuffer.buffer.renderTransparent(renderContext);
+		}
 	}
 
-	private boolean rebuildAllBuffers = false;
-
 	public void MarkAllBuffersDirty()
 	{
-		this.rebuildAllBuffers = true;
+		rebuildAllBuffers.set(true);
 	}
 
-	public void updateQuadTreeRenderSources()
-	{
-		boolean rebuildAllBuffers = this.rebuildAllBuffers;
-		this.rebuildAllBuffers = false;
-
-		Iterator<QuadNode<LodRenderSection>> nodeIterator = this.lodQuadTree.nodeIterator();
-		while (nodeIterator.hasNext())
-		{
-			LodRenderSection renderSection = nodeIterator.next().value;
-			try {
-				if (renderSection != null)
-				{
-					if (rebuildAllBuffers)
-					{
-						renderSection.markBufferDirty();
-					}
-					renderSection.tryBuildAndSwapBuffer();
-				}
-			}
-			catch (Exception e)
-			{
-				LOGGER.error("Error updating QuadTree render source at "+renderSection.pos+".", e);
-			}
-		}
-	}
 	
     public void close() 
 	{

core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java

@@ -228,7 +228,7 @@ public class LodRenderer
 		//GL32.glEnable( GL32.GL_POLYGON_OFFSET_FILL );
 		//GL32.glPolygonOffset( 1f, 1f );
 
-		bufferHandler.buildRenderList(this.getLookVector());
+		bufferHandler.buildRenderListAndUpdateSections(this.getLookVector());
 
 		//===========//
 		// rendering //
@@ -253,7 +253,7 @@ public class LodRenderer
 		if (LodRenderer.transparencyEnabled) {
 			GL32.glEnable(GL32.GL_BLEND);
 			GL32.glBlendFunc(GL32.GL_SRC_ALPHA, GL32.GL_ONE_MINUS_SRC_ALPHA);
-			GL32.glDepthMask(false); // This so that even on incorrect sorting of transparent blocks, it still mostly looks correct
+			//GL32.glDepthMask(false); // This so that even on incorrect sorting of transparent blocks, it still mostly looks correct
 			bufferHandler.renderTransparent(this);
 			GL32.glDepthMask(true); // Apparently the depth mask state is stored in the FBO, so glState fails to restore it...
 		}

core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java

@@ -2,14 +2,17 @@ package com.seibel.distanthorizons.core.util;
 
 import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
 import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider;
+import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler;
 import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
 import org.apache.logging.log4j.Logger;
 
 import javax.annotation.Nullable;
+import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -19,6 +22,7 @@ public class FileScanUtil
     private static final Logger LOGGER = DhLoggerBuilder.getLogger();
     public static final int MAX_SCAN_DEPTH = 5;
     public static final String LOD_FILE_POSTFIX = ".lod";
+	public static final String RENDER_FILE_POSTFIX = ".rlod";
 	
     public static void scanFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper,
 			@Nullable IFullDataSourceProvider dataSourceProvider,
@@ -28,10 +32,11 @@ public class FileScanUtil
 		{
 			try (Stream<Path> pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
 			{
-				dataSourceProvider.addScannedFile(pathStream.filter(
-								path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile()
-						).map(Path::toFile).collect(Collectors.toList())
-				);
+				List<File> files = pathStream.filter(
+						path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile()
+				).map(Path::toFile).collect(Collectors.toList());
+				LOGGER.info("Found "+files.size()+" full data files for "+levelWrapper+" in "+saveStructure);
+				dataSourceProvider.addScannedFile(files);
 			}
 			catch (Exception e)
 			{
@@ -43,10 +48,11 @@ public class FileScanUtil
 		{
 			try (Stream<Path> pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
 			{
-				renderSourceProvider.addScannedFile(pathStream.filter((
-								path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile())
-						).map(Path::toFile).collect(Collectors.toList())
-				);
+				List<File> files = pathStream.filter(
+						path -> path.toFile().getName().endsWith(RENDER_FILE_POSTFIX) && path.toFile().isFile()
+				).map(Path::toFile).collect(Collectors.toList());
+				LOGGER.info("Found "+files.size()+" render cache files for "+levelWrapper+" in "+saveStructure);
+				renderSourceProvider.addScannedFile(files);
 			}
 			catch (Exception e)
 			{

core/src/main/java/com/seibel/distanthorizons/core/util/objects/SortedArraySet.java

@@ -67,6 +67,10 @@ public class SortedArraySet<E> implements SortedSet<E> {
         return list.listIterator();
     }
 
+    public ListIterator<E> listIterator(int index) {
+        return list.listIterator(index);
+    }
+
 
     @Override
     public Object[] toArray() {