Merge branch 'main' of https://gitlab.com/jeseibel/distant-horizons-core into refactor/remove-tcp-connection

core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java

@@ -73,13 +73,14 @@ public class ClientApi
 	public static boolean prefLoggerEnabled = false;
 	
 	public static final ClientApi INSTANCE = new ClientApi();
-	public static TestRenderer testRenderer = new TestRenderer();
+	public static final TestRenderer TEST_RENDERER = new TestRenderer();
 	
 	private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
 	
 	public static final long SPAM_LOGGER_FLUSH_NS = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS);
 	
 	private boolean configOverrideReminderPrinted = false;
+	private boolean lowMemoryWarningPrinted = false;
 	
 	private final Queue<String> chatMessageQueueForNextFrame = new LinkedBlockingQueue<>();
 	
@@ -357,29 +358,7 @@ public class ClientApi
 	{
 		// logging //
 		
-		// dev build
-		if (ModInfo.IS_DEV_BUILD && !this.configOverrideReminderPrinted && MC.playerExists())
-		{
-			this.configOverrideReminderPrinted = true;
-			
-			// remind the user that this is a development build
-			MC.sendChatMessage("Distant Horizons nightly/unstable build, version: [" + ModInfo.VERSION+"].");
-			MC.sendChatMessage("Issues may occur with this version.");
-			MC.sendChatMessage("Here be dragons!");
-			MC.sendChatMessage("");
-		}
-		
-		// generic messages
-		while (!this.chatMessageQueueForNextFrame.isEmpty())
-		{
-			String message = this.chatMessageQueueForNextFrame.poll();
-			if (message == null)
-			{
-				// done to prevent potential null pointers
-				message = "";
-			}
-			MC.sendChatMessage(message);
-		}
+		this.sendChatMessagesNow();
 		
 		IProfilerWrapper profiler = MC.getProfiler();
 		profiler.pop(); // get out of "terrain"
@@ -466,7 +445,7 @@ public class ClientApi
 				else if (Config.Client.Advanced.Debugging.rendererMode.get() == EDhApiRendererMode.DEBUG)
 				{
 					profiler.push("Render Debug");
-					ClientApi.testRenderer.render();
+					ClientApi.TEST_RENDERER.render();
 					profiler.pop();
 				}
 			}
@@ -546,6 +525,53 @@ public class ClientApi
 		}
 	}
 	
+	private void sendChatMessagesNow()
+	{
+		// dev build
+		if (ModInfo.IS_DEV_BUILD && !this.configOverrideReminderPrinted && MC.playerExists())
+		{
+			this.configOverrideReminderPrinted = true;
+			
+			// remind the user that this is a development build
+			MC.sendChatMessage("\u00A72" + "Distant Horizons: nightly/unstable build, version: [" + ModInfo.VERSION+"]." + "\u00A7r");
+			MC.sendChatMessage("Issues may occur with this version.");
+			MC.sendChatMessage("Here be dragons!");
+			MC.sendChatMessage("");
+		}
+		
+		// memory
+		if (!this.lowMemoryWarningPrinted && Config.Client.Advanced.Logging.showLowMemoryWarningOnStartup.get())
+		{
+			this.lowMemoryWarningPrinted = true;
+			
+			// 4 GB
+			long minimumRecommendedMemoryInBytes = 4L * 1_000_000_000L;
+			
+			// Java returned 17,171,480,576 for 16 GB so it might be slightly off what you'd expect
+			long maxMemoryInBytes = Runtime.getRuntime().maxMemory();
+			if (maxMemoryInBytes < minimumRecommendedMemoryInBytes)
+			{
+				MC.sendChatMessage("\u00A76" + "Distant Horizons: Low memory detected." + "\u00A7r");
+				MC.sendChatMessage("Stuttering or low FPS may occur.");
+				MC.sendChatMessage("Please increase Minecraft's available memory to 4 gigabytes.");
+				MC.sendChatMessage("This can be disabled in DH's config under Advanced -> Logging.");
+				MC.sendChatMessage("");
+			}
+		}
+		
+		// generic messages
+		while (!this.chatMessageQueueForNextFrame.isEmpty())
+		{
+			String message = this.chatMessageQueueForNextFrame.poll();
+			if (message == null)
+			{
+				// done to prevent potential null pointers
+				message = "";
+			}
+			MC.sendChatMessage(message);
+		}
+	}
+	
 	/** 
 	 * Queues the given message to appear in chat the next valid frame.
 	 * Useful for queueing up messages that may be triggered before the user has loaded into the world. 

core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java

@@ -41,10 +41,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.ArrayList;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -68,23 +65,13 @@ public class SharedApi
 	private static int lastWorldGenTickDelta = 0;
 	private static long lastOverloadedLogMessageMsTime = 0;
 	
-	public F3Screen.DynamicMessage f3Message;
-	
 	
 	
 	//=============//
 	// constructor //
 	//=============//
 	
-	private SharedApi() 
-	{
-		this.f3Message = new F3Screen.DynamicMessage(() ->
-		{
-			int maxUpdateCount = MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get();
-			return LodUtil.formatLog("Queued chunk updates: " + UPDATING_CHUNK_POS_SET.size() + " / " + maxUpdateCount);
-		});
-	}
-	
+	private SharedApi() { }
 	public static void init() { Initializer.init(); }
 	
 	
@@ -228,7 +215,7 @@ public class SharedApi
 			if (msBetweenLastLog >= MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE)
 			{
 				lastOverloadedLogMessageMsTime = System.currentTimeMillis();
-				LOGGER.warn("Too many chunks queued for updating, max queue count ["+maxQueueCount+"] (["+MAX_UPDATING_CHUNK_COUNT_PER_THREAD+"] per thread). Some LODs may not be updated or may be missing. Please move through the world slower, decrease your vanilla render distance, or increase the CPU load config.");
+				LOGGER.warn("Too many chunks queued for updating, max queue count ["+maxQueueCount+"] (["+MAX_UPDATING_CHUNK_COUNT_PER_THREAD+"] per thread). This may result in holes in your LODs. Please move through the world slower, decrease your vanilla render distance, slow down your world pre-generator, or increase the CPU load config.");
 			}
 			
 			return;
@@ -307,6 +294,23 @@ public class SharedApi
 				
 				try
 				{
+					// check if this chunk has been converted into an LOD already
+					int oldChunkHash = dhLevel.getChunkHash(chunkWrapper.getChunkPos()); // shouldn't happen on the render thread since it may take a few moments to run
+					int newChunkHash = chunkWrapper.getBlockBiomeHashCode();
+					if (oldChunkHash == newChunkHash)
+					{
+						// if the chunk hashes are the same then we don't need to bother with lighting the chunk
+						// or creating/updating the LODs
+						//LOGGER.info("skipping: "+chunkWrapper.getChunkPos()+" "+newChunkHash);
+						return;
+					}
+					else
+					{
+						//LOGGER.info("g: "+chunkWrapper.getChunkPos()+" "+newChunkHash);
+					}
+					
+					
+					
 					// Save or populate the chunk wrapper's lighting
 					// this is done so we don't have to worry about MC unloading the lighting data for this chunk
 					boolean onlyUseDhLighting = Config.Client.Advanced.LodBuilding.onlyUseDhLightingEngine.get();
@@ -315,7 +319,7 @@ public class SharedApi
 						try
 						{
 							// If MC's lighting engine isn't thread safe this may cause the server thread to lag
-							chunkWrapper.bakeDhLightingUsingMcLightingEngine();
+							chunkWrapper.bakeDhLightingUsingMcLightingEngine(); // TODO handle unlit chunks, would pulling in the chunk from disk be a good idea? Look at ChunkLoader in the world gen code for an example
 						}
 						catch (IllegalStateException e)
 						{
@@ -341,6 +345,7 @@ public class SharedApi
 					}
 					
 					dhLevel.updateChunkAsync(chunkWrapper);
+					dhLevel.setChunkHash(chunkWrapper.getChunkPos(), newChunkHash);
 				}
 				catch (Exception e)
 				{
@@ -371,4 +376,18 @@ public class SharedApi
 	}
 	
 	
+	
+	//=========//
+	// F3 Menu //
+	//=========//
+	
+	public String getDebugMenuString()
+	{
+		int maxUpdateCount = MAX_UPDATING_CHUNK_COUNT_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads.get();
+		String updatingCountStr = F3Screen.NUMBER_FORMAT.format(UPDATING_CHUNK_POS_SET.size());
+		String maxUpdateCountStr = F3Screen.NUMBER_FORMAT.format(maxUpdateCount);
+		return "Queued chunk updates: "+updatingCountStr+" / "+maxUpdateCountStr;
+	}
+	
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/config/Config.java

@@ -809,12 +809,12 @@ public class Config
 								+ "")
 						.build();
 				
-				public static ConfigEntry<Boolean> showMigrationChatWarning = new ConfigEntry.Builder<Boolean>()
-						.set(true)
-						.comment(""
-								+ "Determines if a message should be displayed in the chat when LOD migration starts. \n"
-								+ "")
-						.build();
+				//public static ConfigEntry<Boolean> showMigrationChatWarning = new ConfigEntry.Builder<Boolean>()
+				//		.set(true)
+				//		.comment(""
+				//				+ "Determines if a message should be displayed in the chat when LOD migration starts. \n"
+				//				+ "")
+				//		.build();
 				
 			}
 			
@@ -1242,6 +1242,14 @@ public class Config
 								+ "This can be useful for debugging.")
 						.build();
 				
+				
+				public static ConfigEntry<Boolean> showLowMemoryWarningOnStartup = new ConfigEntry.Builder<Boolean>()
+						.set(true)
+						.comment(""
+								+ "If enabled, a chat message will be displayed if Java doesn't have enough \n"
+								+ "memory allocated to run DH well.")
+						.build();
+				
 			}
 			
 			public static class Debugging

core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java

@@ -37,28 +37,27 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan
 	public static final ThreadPresetConfigEventHandler INSTANCE = new ThreadPresetConfigEventHandler();
 	
 	private static final Logger LOGGER = LogManager.getLogger();
-	private static final boolean LOW_THREAD_COUNT_CPU = (Runtime.getRuntime().availableProcessors() <= 4);
 	
 	
 	
-	public static int getWorldGenDefaultThreadCount() { return getThreadCountByPercent(0.15); }
+	public static int getWorldGenDefaultThreadCount() { return getThreadCountByPercent(0.1); }
 	private final ConfigEntryWithPresetOptions<EDhApiThreadPreset, Integer> worldGenThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads,
 			new HashMap<EDhApiThreadPreset, Integer>()
 			{{
 				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 1);
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getWorldGenDefaultThreadCount());
-				this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.25));
-				this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.5));
-				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0));
+				this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.15));
+				this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.25));
+				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(0.5));
 			}});
-	public static double getWorldGenDefaultRunTimeRatio() { return LOW_THREAD_COUNT_CPU ? 0.5 : 0.75; }
+	public static double getWorldGenDefaultRunTimeRatio() { return 0.5; }
 	private final ConfigEntryWithPresetOptions<EDhApiThreadPreset, Double> worldGenRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads,
 			new HashMap<EDhApiThreadPreset, Double>()
 			{{
-				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, LOW_THREAD_COUNT_CPU ? 0.1 : 0.25);
+				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.1);
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getWorldGenDefaultRunTimeRatio());
-				this.put(EDhApiThreadPreset.BALANCED, LOW_THREAD_COUNT_CPU ? 0.5 : 0.75);
-				this.put(EDhApiThreadPreset.AGGRESSIVE, LOW_THREAD_COUNT_CPU ? 0.75 : 1.0);
+				this.put(EDhApiThreadPreset.BALANCED, 0.75);
+				this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0);
 				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0);
 			}});
 	
@@ -71,13 +70,13 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getFileHandlerDefaultThreadCount());
 				this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.2));
 				this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2));
-				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0));
+				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(0.5));
 			}});
-	public static double getFileHandlerDefaultRunTimeRatio() { return 0.75; }
+	public static double getFileHandlerDefaultRunTimeRatio() { return 0.5; }
 	private final ConfigEntryWithPresetOptions<EDhApiThreadPreset, Double> fileHandlerRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads,
 			new HashMap<EDhApiThreadPreset, Double>()
 			{{
-				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.50);
+				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.25);
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getFileHandlerDefaultRunTimeRatio());
 				this.put(EDhApiThreadPreset.BALANCED, 1.0);
 				this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0);
@@ -85,24 +84,24 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan
 			}});
 	
 	
-	public static int getUpdatePropagatorDefaultThreadCount() { return getThreadCountByPercent(0.25); }
+	public static int getUpdatePropagatorDefaultThreadCount() { return getThreadCountByPercent(0.10); }
 	private final ConfigEntryWithPresetOptions<EDhApiThreadPreset, Integer> UpdatePropagatorThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfUpdatePropagatorThreads,
 			new HashMap<EDhApiThreadPreset, Integer>()
 			{{
 				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 1);
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultThreadCount());
-				this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.5));
-				this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.75));
-				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0));
+				this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.25));
+				this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.50));
+				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(0.75));
 			}});
-	public static double getUpdatePropagatorDefaultRunTimeRatio() { return 0.5; }
+	public static double getUpdatePropagatorDefaultRunTimeRatio() { return 0.25; }
 	private final ConfigEntryWithPresetOptions<EDhApiThreadPreset, Double> UpdatePropagatorRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads,
 			new HashMap<EDhApiThreadPreset, Double>()
 			{{
-				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.25);
+				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.1);
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultRunTimeRatio());
-				this.put(EDhApiThreadPreset.BALANCED, 0.75);
-				this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0);
+				this.put(EDhApiThreadPreset.BALANCED, 0.50);
+				this.put(EDhApiThreadPreset.AGGRESSIVE, 0.75);
 				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0);
 			}});
 	
@@ -115,16 +114,16 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getLodBuilderDefaultThreadCount());
 				this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.2));
 				this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4));
-				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0));
+				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(0.6));
 			}});
-	public static double getLodBuilderDefaultRunTimeRatio() { return LOW_THREAD_COUNT_CPU ? 0.25 : 0.5; }
+	public static double getLodBuilderDefaultRunTimeRatio() { return 0.25; }
 	private final ConfigEntryWithPresetOptions<EDhApiThreadPreset, Double> lodBuilderRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForLodBuilderThreads,
 			new HashMap<EDhApiThreadPreset, Double>()
 			{{
 				this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.1);
 				this.put(EDhApiThreadPreset.LOW_IMPACT, getLodBuilderDefaultRunTimeRatio());
-				this.put(EDhApiThreadPreset.BALANCED, LOW_THREAD_COUNT_CPU ? 0.5 : 0.75);
-				this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0);
+				this.put(EDhApiThreadPreset.BALANCED, 0.5);
+				this.put(EDhApiThreadPreset.AGGRESSIVE, 0.75);
 				this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0);
 			}});
 	

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java

@@ -75,7 +75,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
 	/** A flattened 2D array (for the X and Z directions) containing an array for the Y direction. */
 	private final long[][] dataArrays;
 	
-	private long sectionPos;
+	private long pos;
 	
 	private boolean isEmpty = true;
 	
@@ -86,11 +86,11 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
 	//==============//
 	
 	public static FullDataSourceV1 createEmpty(long pos) { return new FullDataSourceV1(pos); }
-	private FullDataSourceV1(long sectionPos)
+	private FullDataSourceV1(long pos)
 	{
 		this.dataArrays = new long[WIDTH * WIDTH][0];
-		this.mapping = new FullDataPointIdMap(sectionPos);
-		this.sectionPos = sectionPos;
+		this.mapping = new FullDataPointIdMap(pos);
+		this.pos = pos;
 	}
 	
 	
@@ -111,19 +111,21 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
 	//=====================//
 	
 	@Override
-	public Long getKey() { return this.sectionPos; }
+	public Long getKey() { return this.pos; }
+	@Override
+	public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
 	
 	@Override
-	public Long getPos() { return this.sectionPos; }
+	public Long getPos() { return this.pos; }
 	
 	public void resizeDataStructuresForRepopulation(long pos)
 	{
 		// no data structures need to be changed, only the source's position 
-		this.sectionPos = pos;
+		this.pos = pos;
 	}
 	
 	@Override
-	public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.sectionPos) - SECTION_SIZE_OFFSET); }
+	public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - SECTION_SIZE_OFFSET); }
 	
 	public boolean isEmpty() { return this.isEmpty; }
 	
@@ -370,7 +372,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
 			throw new IOException("Invalid data content end guard for ID mapping");
 		}
 		
-		return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper);
+		return FullDataPointIdMap.deserialize(inputStream, this.pos, levelWrapper);
 	}
 	public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
 	

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java

@@ -74,6 +74,8 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
 	private long pos;
 	@Override
 	public Long getKey() { return this.pos; }
+	@Override
+	public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
 	
 	
 	public final FullDataPointIdMap mapping;

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java

@@ -218,6 +218,8 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
 	public Long getPos() { return this.pos; }
 	@Override
 	public Long getKey() { return this.pos; }
+	@Override
+	public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
 	
 	public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - SECTION_SIZE_OFFSET); }
 	

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java

@@ -95,6 +95,7 @@ public class FullDataSourceProviderV2
 	
 	protected long legacyDeletionCount = -1;
 	protected long migrationCount = -1;
+	protected boolean migrationStoppedWithError = false;
 	
 	/**
 	 * Tracks which positions are currently being updated
@@ -447,38 +448,39 @@ public class FullDataSourceProviderV2
 		{
 			this.showMigrationStartMessage();
 			
-			
-			// keep going until every data source has been migrated
-			int progressCount = 0;
-			while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get())
+			try
 			{
-				LOGGER.info("Migrating [" + dimensionName + "] - [" + progressCount + "/" + totalMigrationCount + "]...");
-				
-				ArrayList<CompletableFuture<Void>> updateFutureList = new ArrayList<>();
-				for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++)
+				// keep going until every data source has been migrated
+				int progressCount = 0;
+				while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get())
 				{
-					FullDataSourceV1 legacyDataSource = legacyDataSourceList.get(i);
+					LOGGER.info("Migrating [" + dimensionName + "] - [" + progressCount + "/" + totalMigrationCount + "]...");
 					
-					try
+					ArrayList<CompletableFuture<Void>> updateFutureList = new ArrayList<>();
+					for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++)
 					{
-						// convert the legacy data source to the new format,
-						// this is a relatively cheap operation
-						FullDataSourceV2 newDataSource = FullDataSourceV2.createFromLegacyDataSourceV1(legacyDataSource);
-						newDataSource.applyToParent = true;
+						FullDataSourceV1 legacyDataSource = legacyDataSourceList.get(i);
 						
-						// the actual update process can be moderately expensive due to having to update
-						// the render data along with the full data, so running it async on the update threads gains us a good bit of speed
-						CompletableFuture<Void> future = this.updateDataSourceAsync(newDataSource);
-						updateFutureList.add(future);
-						future.thenRun(() ->
+						try
 						{
-							// after the update finishes the legacy data source can be safely deleted
-							this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos());
+							// convert the legacy data source to the new format,
+							// this is a relatively cheap operation
+							FullDataSourceV2 newDataSource = FullDataSourceV2.createFromLegacyDataSourceV1(legacyDataSource);
+							newDataSource.applyToParent = true;
 							
-							try
+							// the actual update process can be moderately expensive due to having to update
+							// the render data along with the full data, so running it async on the update threads gains us a good bit of speed
+							CompletableFuture<Void> future = this.updateDataSourceAsync(newDataSource);
+							updateFutureList.add(future);
+							future.thenRun(() ->
 							{
-								newDataSource.close();
-							}
+								// after the update finishes the legacy data source can be safely deleted
+								this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos());
+								
+								try
+								{
+									newDataSource.close();
+								}
 							catch (Exception ignore)
 							{
 							}
@@ -513,8 +515,15 @@ public class FullDataSourceProviderV2
 				progressCount += legacyDataSourceList.size();
 				this.migrationCount -= legacyDataSourceList.size();
 			}
-			
-			
+			}
+			catch (Exception e)
+			{
+				LOGGER.info("migration stopped due to error for: ["+dimensionName+"]-["+this.saveDir+"], error: ["+e.getMessage()+"].", e);
+				this.showMigrationEndMessage(false);
+				this.migrationStoppedWithError = true;
+			}
+			finally
+			{
 			if (this.migrationThreadRunning.get())
 			{
 				LOGGER.info("migration complete for: [" + dimensionName + "]-[" + this.saveDir + "].");
@@ -525,6 +534,8 @@ public class FullDataSourceProviderV2
 			{
 				LOGGER.info("migration stopped for: [" + dimensionName + "]-[" + this.saveDir + "].");
 				this.showMigrationEndMessage(false);
+					this.migrationStoppedWithError = true;
+				}
 			}
 		}
 		else
@@ -537,6 +548,7 @@ public class FullDataSourceProviderV2
 	
 	public long getLegacyDeletionCount() { return this.legacyDeletionCount; }
 	public long getTotalMigrationCount() { return this.migrationCount; }
+	public boolean getMigrationStoppedWithError() { return this.migrationStoppedWithError; }
 	
 	
 	private void showMigrationStartMessage()

core/src/main/java/com/seibel/distanthorizons/core/file/structure/AbstractSaveStructure.java

@@ -45,8 +45,6 @@ public abstract class AbstractSaveStructure implements AutoCloseable
 	 */
 	public abstract File getLevelFolder(ILevelWrapper wrapper);
 	
-	/** Will return null if no parent folder exists for the given {@link ILevelWrapper}. */
-	public abstract File getRenderCacheFolder(ILevelWrapper world);
 	/** Will return null if no parent folder exists for the given {@link ILevelWrapper}. */
 	public abstract File getFullDataFolder(ILevelWrapper world);
 	

core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java

@@ -168,18 +168,6 @@ public class ClientOnlySaveStructure extends AbstractSaveStructure
 	}
 	
 	
-	@Override
-	public File getRenderCacheFolder(ILevelWrapper level)
-	{
-		File levelFolder = this.levelWrapperToFileMap.get(level);
-		if (levelFolder == null)
-		{
-			return null;
-		}
-		
-		return levelFolder;
-	}
-	
 	@Override
 	public File getFullDataFolder(ILevelWrapper level)
 	{

core/src/main/java/com/seibel/distanthorizons/core/file/structure/LocalSaveStructure.java

@@ -51,14 +51,6 @@ public class LocalSaveStructure extends AbstractSaveStructure
 		return serverSide.getSaveFolder();
 	}
 	
-	@Override
-	public File getRenderCacheFolder(ILevelWrapper level)
-	{
-		IServerLevelWrapper serverSide = (IServerLevelWrapper) level;
-		this.debugPath = serverSide.getSaveFolder();
-		return serverSide.getSaveFolder();
-	}
-	
 	@Override
 	public File getFullDataFolder(ILevelWrapper level)
 	{

core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java

@@ -33,6 +33,7 @@ import java.util.*;
 import java.util.concurrent.locks.ReentrantLock;
 
 import it.unimi.dsi.fastutil.ints.IntArrayList;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * This logic was roughly based on
@@ -67,7 +68,7 @@ public class DhLightingEngine
 	 * @param nearbyChunkList should also contain centerChunk
 	 * @param maxSkyLight should be a value between 0 and 15
 	 */
-	public void lightChunk(IChunkWrapper centerChunk, ArrayList<IChunkWrapper> nearbyChunkList, int maxSkyLight)
+	public void lightChunk(@NotNull IChunkWrapper centerChunk, @NotNull ArrayList<IChunkWrapper> nearbyChunkList, int maxSkyLight)
 	{
 		DhChunkPos centerChunkPos = centerChunk.getChunkPos();
 		AdjacentChunkHolder adjacentChunkHolder = new AdjacentChunkHolder(centerChunk);
@@ -104,7 +105,6 @@ public class DhLightingEngine
 			for (int chunkIndex = 0; chunkIndex < nearbyChunkList.size(); chunkIndex++) // using iterators in high traffic areas can cause GC issues due to allocating a bunch of iterators, use an indexed for-loop instead
 			{
 				IChunkWrapper chunk = nearbyChunkList.get(chunkIndex);
-				
 				if (chunk != null && requestedAdjacentPositions.contains(chunk.getChunkPos()))
 				{
 					// remove the newly found position
@@ -179,7 +179,7 @@ public class DhLightingEngine
 					break;
 				}
 			}
-
+			
 			// block light
 			this.propagateLightPosList(blockLightPosQueue, adjacentChunkHolder,
 					(neighbourChunk, relBlockPos) -> neighbourChunk.getDhBlockLight(relBlockPos.x, relBlockPos.y, relBlockPos.z),

core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java

@@ -30,6 +30,7 @@ import com.seibel.distanthorizons.core.jar.installer.WebDownloader;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.wrapperInterfaces.IVersionConstants;
 import com.seibel.distanthorizons.coreapi.ModInfo;
+import com.seibel.distanthorizons.coreapi.util.StringUtil;
 import com.seibel.distanthorizons.coreapi.util.jar.DeleteOnUnlock;
 import org.apache.logging.log4j.Logger;
 
@@ -104,7 +105,7 @@ public class SelfUpdater
 		}
 		if (!ModrinthGetter.mcVersions.contains(mcVersion))
 		{
-			LOGGER.warn("Minecraft version ["+ mcVersion +"] is not findable on Modrinth, only findable versions are ["+ ModrinthGetter.mcVersions.toString() +"]");
+			LOGGER.warn("Minecraft version ["+ mcVersion +"] is not findable on Modrinth, only findable versions are ["+ StringUtil.join(",", ModrinthGetter.mcVersions) +"]");
 			return false;
 		}
 		
@@ -151,7 +152,7 @@ public class SelfUpdater
 		
 		if (!GitlabGetter.INSTANCE.getDownloads(pipeline.get("id")).containsKey(mcVersion))
 		{
-			LOGGER.warn("Minecraft version ["+ mcVersion +"] is not findable on Gitlab, findable versions are ["+ GitlabGetter.INSTANCE.getDownloads(pipeline.get("id")).keySet().toArray().toString() +"].");
+			LOGGER.warn("Minecraft version ["+ mcVersion +"] is not findable on Gitlab, findable versions are ["+ StringUtil.join(",", GitlabGetter.INSTANCE.getDownloads(pipeline.get("id")).keySet().toArray()) +"].");
 			return false;
 		}
 		

core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java

@@ -23,18 +23,30 @@ import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiChunkMo
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.dataObjects.transformers.ChunkToLodBuilder;
 import com.seibel.distanthorizons.core.file.fullDatafile.DelayedFullDataSourceSaveCache;
+import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhChunkPos;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import com.seibel.distanthorizons.core.sql.dto.ChunkHashDTO;
+import com.seibel.distanthorizons.core.sql.repo.ChunkHashRepo;
 import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
 import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector;
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 
+import java.sql.SQLException;
 import java.util.HashSet;
 import java.util.concurrent.ConcurrentHashMap;
 
 public abstract class AbstractDhLevel implements IDhLevel
 {
+	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
+	
 	public final ChunkToLodBuilder chunkToLodBuilder;
 	
+	/** if this is null then the other handler is probably null too, but just in case */
+	@Nullable
+	public ChunkHashRepo chunkHashRepo;
+	
 	protected final DelayedFullDataSourceSaveCache delayedFullDataSourceSaveCache = new DelayedFullDataSourceSaveCache(this::onDataSourceSave, 2_000);
 	/** contains the {@link DhChunkPos} for each {@link DhSectionPos} that are queued to save via {@link AbstractDhLevel#delayedFullDataSourceSaveCache} */
 	protected final ConcurrentHashMap<Long, HashSet<DhChunkPos>> updatedChunkPosSetBySectionPos = new ConcurrentHashMap<>();
@@ -47,6 +59,20 @@ public abstract class AbstractDhLevel implements IDhLevel
 	
 	protected AbstractDhLevel() { this.chunkToLodBuilder = new ChunkToLodBuilder(); }
 	
+	protected void createAndSetChunkHashRepo(String databaseFilePath)
+	{
+		ChunkHashRepo newChunkHashRepo = null;
+		try
+		{
+			newChunkHashRepo = new ChunkHashRepo("jdbc:sqlite", databaseFilePath);
+		}
+		catch (SQLException e)
+		{
+			LOGGER.error("Unable to create [ChunkHashRepo], error: ["+e.getMessage()+"].", e);
+		}
+		this.chunkHashRepo = newChunkHashRepo;
+	}
+	
 	
 	
 	//=================//
@@ -99,6 +125,32 @@ public abstract class AbstractDhLevel implements IDhLevel
 	}
 	
 	
+	@Override
+	public int getChunkHash(DhChunkPos pos)
+	{
+		if (this.chunkHashRepo == null)
+		{
+			return 0;
+		}
+		
+		ChunkHashDTO dto = this.chunkHashRepo.getByKey(pos);
+		return (dto != null) ? dto.chunkHash : 0;
+	}
+	@Override
+	public void setChunkHash(DhChunkPos pos, int chunkHash)
+	{
+		if (this.chunkHashRepo != null)
+		{
+			this.chunkHashRepo.save(new ChunkHashDTO(pos, chunkHash));
+		}
+	}
+	
+	
+	
+	//================//
+	// base overrides //
+	//================//
+	
 	@Override
 	public void close() { this.chunkToLodBuilder.close(); }
 	

core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java

@@ -57,8 +57,6 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I
 	public final FullDataSourceProviderV2 fullDataSourceProvider;
 	public final AtomicReference<ClientRenderState> ClientRenderStateRef = new AtomicReference<>();
 	
-	public final F3Screen.NestedMessage f3Message;
-	
 	
 	
 	//=============//
@@ -68,7 +66,6 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I
 	public ClientLevelModule(IDhClientLevel clientLevel)
 	{
 		this.clientLevel = clientLevel;
-		this.f3Message = new F3Screen.NestedMessage(this::f3Log);
 		
 		this.fullDataSourceProvider = this.clientLevel.getFullDataProvider();
 		this.fullDataSourceProvider.dateSourceUpdateListeners.add(this);
@@ -246,8 +243,6 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I
 		}
 		
 		this.fullDataSourceProvider.dateSourceUpdateListeners.remove(this);
-		
-		this.f3Message.close();
 	}
 	
 	
@@ -256,53 +251,6 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I
 	// misc helper functions //
 	//=======================//
 	
-	private String[] f3Log()
-	{
-		String dimName = this.clientLevel.getLevelWrapper().getDimensionType().getDimensionName();
-		boolean rendererActive = this.ClientRenderStateRef.get() != null;
-		
-		ThreadPoolExecutor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor();
-		String fileQueueSize = (fileExecutor != null) ? fileExecutor.getQueue().size()+"" : "-";
-		String fileCompletedTaskSize = (fileExecutor != null) ? fileExecutor.getCompletedTaskCount()+"" : "-";
-		
-		ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor();
-		String updateQueueSize = (updateExecutor != null) ? updateExecutor.getQueue().size()+"" : "-";
-		String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-";
-		
-		int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount();
-		long legacyDeletionCount = this.fullDataSourceProvider.getLegacyDeletionCount();
-		long migrationCount = this.fullDataSourceProvider.getTotalMigrationCount();
-		
-		
-		
-		ArrayList<String> lines = new ArrayList<>();
-		lines.add("");
-		lines.add("level [" + dimName + "] rendering: " + (rendererActive ? "Active" : "Inactive"));
-		// TODO a lot of these items only need to be rendered once, but we don't currently have a way of doing that, so only add them for the rendered level 
-		if (rendererActive)
-		{
-			lines.add("File Handler [" + dimName + "]");
-			lines.add("  File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")");
-			if (legacyDeletionCount > 0)
-			{
-				lines.add("  Legacy Deletion #: " + legacyDeletionCount);
-			}
-			if (migrationCount > 0)
-			{
-				lines.add("  Legacy Migration #: " + migrationCount);
-			}
-			lines.add("  Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")");
-			lines.add("  Level Unsaved #: " + this.clientLevel.getUnsavedDataSourceCount());
-			if (unsavedDataSourceCount != -1)
-			{
-				lines.add("  File Handler Unsaved #: " + unsavedDataSourceCount);
-			}
-			lines.add("  Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size());
-		}
-		
-		return lines.toArray(new String[0]);
-	}
-	
 	public void clearRenderCache()
 	{
 		ClientRenderState ClientRenderState = this.ClientRenderStateRef.get();

core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java

@@ -49,6 +49,8 @@ import org.jetbrains.annotations.Nullable;
 import javax.annotation.CheckForNull;
 import java.awt.*;
 import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
 /** The level used when connected to a server */
@@ -115,6 +117,9 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
 		this.worldGenModule = new WorldGenModule(this);
 		
 		this.clientside = new ClientLevelModule(this);
+		
+		this.createAndSetChunkHashRepo(this.dataFileHandler.repo.databaseLocation);
+		
 		if (enableRendering)
 		{
 			this.clientside.startRenderer(clientLevelWrapper);
@@ -246,6 +251,34 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
 	@Override
 	public int getMinY() { return this.levelWrapper.getMinHeight(); }
 	
+	@Override
+	public void addDebugMenuStringsToList(List<String> messageList)
+	{
+		String dimName = this.levelWrapper.getDimensionType().getDimensionName();
+		boolean rendering = this.clientside.isRendering();
+		messageList.add("["+dimName+"] rendering: "+(rendering ? "yes" : "no"));
+		
+		
+		boolean migrationErrored = this.dataFileHandler.getMigrationStoppedWithError();
+		if (!migrationErrored)
+		{
+			long legacyDeletionCount = this.dataFileHandler.getLegacyDeletionCount();
+			if (legacyDeletionCount > 0)
+			{
+				messageList.add("  Migrating - Deleting #: " + legacyDeletionCount);
+			}
+			long migrationCount = this.dataFileHandler.getTotalMigrationCount();
+			if (migrationCount > 0)
+			{
+				messageList.add("  Migrating - Conversion #: " + migrationCount);
+			}
+		}
+		else
+		{
+			messageList.add("  Migration Failed");
+		}
+	}
+	
 	@Override
 	public void close()
 	{

core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java

@@ -23,6 +23,7 @@ import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhAp
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
 import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
+import com.seibel.distanthorizons.core.logging.f3.F3Screen;
 import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
 import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -38,6 +39,8 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapp
 import org.apache.logging.log4j.Logger;
 
 import java.awt.*;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
 /** The level used on a singleplayer world */
@@ -53,6 +56,10 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev
 	
 	
 	
+	//=============//
+	// constructor //
+	//=============//
+	
 	public DhClientServerLevel(AbstractSaveStructure saveStructure, IServerLevelWrapper serverLevelWrapper)
 	{
 		if (saveStructure.getFullDataFolder(serverLevelWrapper).mkdirs())
@@ -62,6 +69,8 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev
 		this.serverLevelWrapper = serverLevelWrapper;
 		this.serverside = new ServerLevelModule(this, saveStructure);
 		this.clientside = new ClientLevelModule(this);
+		this.createAndSetChunkHashRepo(this.serverside.fullDataFileHandler.repo.databaseLocation);
+		
 		LOGGER.info("Started " + DhClientServerLevel.class.getSimpleName() + " for " + serverLevelWrapper + " with saves at " + saveStructure);
 	}
 	
@@ -72,10 +81,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev
 	//==============//
 	
 	@Override
-	public void clientTick()
-	{
-		clientside.clientTick();
-	}
+	public void clientTick() { this.clientside.clientTick(); }
 	
 	@Override
 	public void render(DhApiRenderParam renderEventParam, IProfilerWrapper profiler)
@@ -124,6 +130,8 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev
 		}
 	}
 	
+	
+	
 	//========//
 	// render //
 	//========//
@@ -132,6 +140,8 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev
 	
 	public void stopRenderer() { this.clientside.stopRenderer(); }
 	
+	
+	
 	//================//
 	// level handling //
 	//================//
@@ -180,7 +190,52 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev
 	public CompletableFuture<Void> updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientside.updateDataSourcesAsync(data); }
 	
 	@Override
-	public int getMinY() { return getLevelWrapper().getMinHeight(); }
+	public int getMinY() { return this.getLevelWrapper().getMinHeight(); }
+	
+	
+	
+	//===========//
+	// debugging //
+	//===========//
+	
+	@Override
+	public void addDebugMenuStringsToList(List<String> messageList)
+	{
+		// header
+		String dimName = this.serverLevelWrapper.getDimensionType().getDimensionName();
+		boolean rendering = this.clientside.isRendering();
+		messageList.add("["+dimName+"] rendering: "+(rendering ? "yes" : "no"));
+		
+		
+		// migration
+		boolean migrationErrored = this.serverside.fullDataFileHandler.getMigrationStoppedWithError();
+		if (!migrationErrored)
+		{
+			long legacyDeletionCount = this.serverside.fullDataFileHandler.getLegacyDeletionCount();
+			if (legacyDeletionCount > 0)
+			{
+				messageList.add("  Migrating - Deleting #: " + F3Screen.NUMBER_FORMAT.format(legacyDeletionCount));
+			}
+			long migrationCount = this.serverside.fullDataFileHandler.getTotalMigrationCount();
+			if (migrationCount > 0)
+			{
+				messageList.add("  Migrating - Conversion #: " + F3Screen.NUMBER_FORMAT.format(migrationCount));
+			}
+		}
+		else
+		{
+			messageList.add("  Migration Failed");
+		}
+		
+		
+		// world gen
+		WorldGenModule worldGenState = this.serverside.worldGenModule;
+		String worldGenDisplayString = worldGenState.getDebugMenuString();
+		if (worldGenDisplayString != null)
+		{
+			messageList.add(worldGenDisplayString);
+		}
+	}
 	
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java

@@ -46,6 +46,8 @@ import com.seibel.distanthorizons.coreapi.util.math.Vec3d;
 import org.apache.logging.log4j.Logger;
 
 import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
 import javax.annotation.CheckForNull;
@@ -61,6 +63,8 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel
 	public final ServerLevelModule serverside;
 	private final IServerLevelWrapper serverLevelWrapper;
 	
+	
+	
 	private final RemotePlayerConnectionHandler remotePlayerConnectionHandler;
 	
 	private final ConcurrentLinkedQueue<IServerPlayerWrapper> worldGenLoopingQueue = new ConcurrentLinkedQueue<>();
@@ -75,6 +79,8 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel
 		}
 		this.serverLevelWrapper = serverLevelWrapper;
 		this.serverside = new ServerLevelModule(this, saveStructure);
+		this.createAndSetChunkHashRepo(this.serverside.fullDataFileHandler.repo.databaseLocation);
+		
 		LOGGER.info("Started DHLevel for {} with saves at {}", serverLevelWrapper, saveStructure);
 	
 		this.remotePlayerConnectionHandler = remotePlayerConnectionHandler;
@@ -172,6 +178,8 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel
 		});
 	}
 	
+	
+	
 	public <T extends PluginChannelMessage> Consumer<T> currentLevelOnly(Consumer<T> next)
 	{
 		return msg ->
@@ -378,6 +386,19 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel
 		}
 	}
 	
+	
+	
+	//===========//
+	// debugging //
+	//===========//
+	
+	@Override
+	public void addDebugMenuStringsToList(List<String> messageList)
+	{
+		String dimName = this.serverLevelWrapper.getDimensionType().getDimensionName();
+		messageList.add("["+dimName+"]");
+	}
+	
 	private static class IncompleteDataSourceEntry
 	{
 		@CheckForNull

core/src/main/java/com/seibel/distanthorizons/core/level/IDhClientLevel.java

@@ -22,7 +22,6 @@ package com.seibel.distanthorizons.core.level;
 import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam;
 import com.seibel.distanthorizons.core.pos.DhBlockPos;
 import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
-import com.seibel.distanthorizons.coreapi.util.math.Mat4f;
 import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper;

core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java

@@ -22,9 +22,11 @@ package com.seibel.distanthorizons.core.level;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
 import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
+import com.seibel.distanthorizons.core.pos.DhChunkPos;
 import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
 
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
 public interface IDhLevel extends AutoCloseable
@@ -37,6 +39,9 @@ public interface IDhLevel extends AutoCloseable
 	 */
 	ILevelWrapper getLevelWrapper();
 	
+	/** @return 0 if no hash is known */
+	int getChunkHash(DhChunkPos pos);
+	void setChunkHash(DhChunkPos pos, int chunkHash);
 	void updateChunkAsync(IChunkWrapper chunk);
 	
 	FullDataSourceProviderV2 getFullDataProvider();
@@ -53,4 +58,7 @@ public interface IDhLevel extends AutoCloseable
 	 */
 	int getUnsavedDataSourceCount();
 	
+	void addDebugMenuStringsToList(List<String> messageList);
+	
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java

@@ -43,6 +43,10 @@ public class ServerLevelModule implements AutoCloseable
 	
 	
 	
+	//=============//
+	// constructor //
+	//=============//
+	
 	public ServerLevelModule(IDhServerLevel parentServerLevel, AbstractSaveStructure saveStructure)
 	{
 		this.parentServerLevel = parentServerLevel;
@@ -54,6 +58,10 @@ public class ServerLevelModule implements AutoCloseable
 	
 	
 	
+	//================//
+	// base overrides //
+	//================//
+	
 	@Override
 	public void close()
 	{

core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java

@@ -37,30 +37,16 @@ public class WorldGenModule implements Closeable
 	private final GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener onWorldGenCompleteListener;
 	
 	private final AtomicReference<AbstractWorldGenState> worldGenStateRef = new AtomicReference<>();
-	private final F3Screen.DynamicMessage worldGenF3Message;
 	
 	
 	
+	//=============//
+	// constructor //
+	//=============//
+	
 	public WorldGenModule(GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener onWorldGenCompleteListener)
 	{
 		this.onWorldGenCompleteListener = onWorldGenCompleteListener;
-		this.worldGenF3Message = new F3Screen.DynamicMessage(() ->
-		{
-			AbstractWorldGenState worldGenState = this.worldGenStateRef.get();
-			if (worldGenState != null)
-			{
-				int waitingCount = worldGenState.worldGenerationQueue.getWaitingTaskCount();
-				int inProgressCount = worldGenState.worldGenerationQueue.getInProgressTaskCount();
-				int totalCountEstimate = worldGenState.worldGenerationQueue.getEstimatedTotalTaskCount();
-				
-				return "World Gen Tasks: "+waitingCount+"/"+totalCountEstimate+", (in progress: "+inProgressCount+")";
-			}
-			else
-			{
-				return "World Gen Disabled";
-			}
-		});
-		
 	}
 	
 	
@@ -114,6 +100,12 @@ public class WorldGenModule implements Closeable
 		}
 	}
 	
+	
+	
+	//=======================//
+	// base method overrides //
+	//=======================//
+	
 	@Override
 	public void close()
 	{
@@ -135,8 +127,6 @@ public class WorldGenModule implements Closeable
 				worldGenState.closeAsync(true).join(); //TODO: Make it async.
 			}
 		}
-		
-		this.worldGenF3Message.close();
 	}
 	
 	
@@ -147,6 +137,22 @@ public class WorldGenModule implements Closeable
 	
 	public boolean isWorldGenRunning() { return this.worldGenStateRef.get() != null; }
 	
+	public String getDebugMenuString()
+	{
+		AbstractWorldGenState worldGenState = this.worldGenStateRef.get();
+		if (worldGenState == null)
+		{
+			return null;
+		}
+		
+		
+		String waitingCountStr = F3Screen.NUMBER_FORMAT.format(worldGenState.worldGenerationQueue.getWaitingTaskCount());
+		String inProgressCountStr = F3Screen.NUMBER_FORMAT.format(worldGenState.worldGenerationQueue.getInProgressTaskCount());
+		String totalCountEstimateStr = F3Screen.NUMBER_FORMAT.format(worldGenState.worldGenerationQueue.getEstimatedTotalTaskCount());
+		
+		return "World Gen Tasks: "+waitingCountStr+"/"+totalCountEstimateStr+" (in progress: "+inProgressCountStr+")";
+	}
+	
 	
 	
 	//================//

core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java

@@ -19,159 +19,127 @@
 
 package com.seibel.distanthorizons.core.logging.f3;
 
+import com.seibel.distanthorizons.core.api.internal.SharedApi;
+import com.seibel.distanthorizons.core.level.IDhLevel;
+import com.seibel.distanthorizons.core.render.RenderBufferHandler;
+import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
+import com.seibel.distanthorizons.core.world.AbstractDhWorld;
 import com.seibel.distanthorizons.coreapi.ModInfo;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 
-import java.io.Closeable;
+import java.lang.ref.WeakReference;
+import java.text.NumberFormat;
 import java.util.*;
-import java.util.function.Supplier;
+import java.util.concurrent.ThreadPoolExecutor;
 
 public class F3Screen
 {
 	private static final Logger LOGGER = LogManager.getLogger();
 	
-	private static final String[] DEFAULT_STRING = {
-			"", // blank line for spacing
-			ModInfo.READABLE_NAME + " version: " + ModInfo.VERSION
-	};
-	private static final List<Message> SELF_UPDATE_MESSAGE_LIST = Collections.synchronizedList(new LinkedList<>());
+	public static final NumberFormat NUMBER_FORMAT = NumberFormat.getIntegerInstance();
 	
-	public static void addStringToDisplay(List<String> list)
+	
+	
+	//============//
+	// properties //
+	//============//
+	
+	private static WeakReference<RenderBufferHandler> renderBufferHandlerRef = new WeakReference<>(null);
+	public static void setRenderBufferHandler(@Nullable RenderBufferHandler renderBufferHandler)
 	{
-		list.addAll(Arrays.asList(DEFAULT_STRING));
-		synchronized (SELF_UPDATE_MESSAGE_LIST)
+		if (renderBufferHandler != null && renderBufferHandlerRef.get() != null)
 		{
-			Iterator<Message> iterator = SELF_UPDATE_MESSAGE_LIST.iterator();
-			while (iterator.hasNext())
+			LOGGER.warn("multiple RenderBufferHandlers are active at once, the F3 menu may not be accurate.");
+		}
+		
+		renderBufferHandlerRef = new WeakReference<>(renderBufferHandler);
+	}
+	
+	
+	
+	//=================//
+	// injection point //
+	//=================//
+	
+	/**
+	 * F3 menu example: <br>
+	 <code>
+	 Distant Horizons v: 2.1.1-a-dev <br><br>
+	 
+	 Queued chunk updates: 0 / 1000 <br>
+	 World Gen Tasks: 40/5304, (in progress: 7) <br><br>
+	 
+	 File thread pool tasks: 0 (complete: 759) <br>
+	 Update thread pool tasks: 10 (complete: 24) <br>
+	 Level Unsaved #: 0 <br>
+	 File Handler Unsaved #: 0 <br>
+	 Parent Update #: 12 <br><br>
+	 
+	 Client_Server World with 3 levels <br>
+	 [overworld] rendering: Active <br>
+	 [the_end] rendering: Inactive <br>
+	 [the_nether] rendering: Inactive <br><br>
+	 
+	 VBO Render Count: 199/374 <br>
+	 </code>
+	 */
+	public static void addStringToDisplay(List<String> messageList)
+	{
+		ThreadPoolExecutor worldGenPool = ThreadPoolUtil.getWorldGenExecutor();
+		ThreadPoolExecutor fileHandlerPool = ThreadPoolUtil.getFileHandlerExecutor();
+		ThreadPoolExecutor updatePool = ThreadPoolUtil.getUpdatePropagatorExecutor();
+		
+		AbstractDhWorld world = SharedApi.getAbstractDhWorld();
+		Iterable<? extends IDhLevel> levelIterator = world.getAllLoadedLevels();
+		
+		
+		messageList.add("");
+		messageList.add(ModInfo.READABLE_NAME+": "+ModInfo.VERSION);
+		messageList.add("");
+		// thread pools
+		messageList.add(getThreadPoolStatString("World Gen", worldGenPool));//"World Gen Tasks: 40/5304, (in progress: 7)");
+		messageList.add(getThreadPoolStatString("File Handler", fileHandlerPool));
+		messageList.add(getThreadPoolStatString("Update Propagator", updatePool));
+		messageList.add("");
+		// chunk updates
+		messageList.add(SharedApi.INSTANCE.getDebugMenuString());
+		messageList.add("");
+		// rendering
+		RenderBufferHandler renderBufferHandler = renderBufferHandlerRef.get();
+		if (renderBufferHandler != null)
+		{
+			messageList.add(renderBufferHandler.getVboRenderDebugMenuString());
+			String showPassString = renderBufferHandler.getShadowPassRenderDebugMenuString();
+			if (showPassString != null)
 			{
-				Message message = iterator.next();
-				if (message == null)
-				{
-					iterator.remove();
-				}
-				else
-				{
-					message.printTo(list);
-				}
+				messageList.add(showPassString);
 			}
+			messageList.add("");
+		}
+		// world / levels
+		messageList.add(world.GetDebugMenuString());
+		for (IDhLevel level : levelIterator)
+		{
+			level.addDebugMenuStringsToList(messageList);
 		}
 	}
 	
 	
 	
 	//================//
-	// helper classes //
+	// helper methods //
 	//================//
 	
-	// we are using Closeable instead of AutoCloseable because the close method should never throw exceptions
-	// and because this class shouldn't be used in a try {} block.
-	public static abstract class Message implements Closeable
+	private static String getThreadPoolStatString(String name, ThreadPoolExecutor pool)
 	{
-		protected Message()
-		{
-			SELF_UPDATE_MESSAGE_LIST.add(this);
-		}
-		
-		public abstract void printTo(List<String> output);
-		
-		@Override
-		public void close()
-		{
-			boolean removed = SELF_UPDATE_MESSAGE_LIST.remove(this);
-		}
+		String queueSize = (pool != null) ? NUMBER_FORMAT.format(pool.getQueue().size()) : "-";
+		String completedCount = (pool != null) ? NUMBER_FORMAT.format(pool.getCompletedTaskCount()) : "-";
 		
+		return name+", tasks: "+queueSize+", complete: "+completedCount;
 	}
 	
-	public static class StaticMessage extends Message
-	{
-		private final String[] lines;
-		
-		public StaticMessage(String... lines) { this.lines = lines; }
-		
-		@Override
-		public void printTo(List<String> output) { output.addAll(Arrays.asList(this.lines)); }
-		
-	}
 	
-	public static class DynamicMessage extends Message
-	{
-		private final Supplier<String> supplier;
-		
-		public DynamicMessage(Supplier<String> message) { this.supplier = message; }
-		
-		public void printTo(List<String> list)
-		{
-			
-			try
-			{
-				String message = this.supplier.get();
-				if (message != null)
-				{
-					list.add(message);
-				}
-			}
-			catch (Exception e)
-			{
-				LOGGER.error("Unexpected Exception in F3 ["+DynamicMessage.class.getSimpleName()+"], error: "+e.getMessage(), e);
-			}
-		}
-		
-	}
-	
-	public static class MultiDynamicMessage extends Message
-	{
-		private final Supplier<String>[] supplierList;
-		
-		@SafeVarargs
-		public MultiDynamicMessage(Supplier<String>... suppliers) { this.supplierList = suppliers; }
-		
-		public void printTo(List<String> list)
-		{
-			for (Supplier<String> supplier : this.supplierList)
-			{
-				try
-				{
-					String message = supplier.get();
-					if (message != null)
-					{
-						list.add(message);
-					}
-				}
-				catch (Exception e)
-				{
-					LOGGER.error("Unexpected Exception in F3 ["+DynamicMessage.class.getSimpleName()+"], error: "+e.getMessage(), e);
-				}
-			}
-		}
-		
-	}
-	
-	public static class NestedMessage extends Message
-	{
-		private final Supplier<String[]> supplier;
-		
-		public NestedMessage(Supplier<String[]> message)
-		{
-			this.supplier = message;
-		}
-		
-		public void printTo(List<String> list)
-		{
-			try
-			{
-				String[] message = this.supplier.get();
-				if (message != null)
-				{
-					list.addAll(Arrays.asList(message));
-				}
-			}
-			catch (Exception e)
-			{
-				LOGGER.error("Unexpected Exception in F3 ["+DynamicMessage.class.getSimpleName()+"], error: "+e.getMessage(), e);
-			}
-		}
-		
-	}
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/network/exceptions/SessionClosedException.java

@@ -0,0 +1,5 @@
+package com.seibel.distanthorizons.core.network.exceptions;
+
+public class SessionClosedException {
+
+}

core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java

@@ -302,13 +302,10 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 			boolean allChildrenSectionsAreLoaded = true;
 			
 			// recursively update all child render sections
-			LongIterator childPosIterator = quadNode.getChildPosIterator();
-			while (childPosIterator.hasNext())
+			for (int i = 0; i < 4; i++)
 			{
-				long childPos = childPosIterator.nextLong();
-				QuadNode<LodRenderSection> childNode = rootNode.getNode(childPos);
-				
-				boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, thisPosIsRendering || parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading);
+				QuadNode<LodRenderSection> childNode = quadNode.getChildByIndex(i);
+				boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, DhSectionPos.getChildByIndex(sectionPos, i), thisPosIsRendering || parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading);
 				allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded;
 			}
 			
@@ -335,13 +332,10 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 				renderSection.renderingEnabled = false;
 				
 				// walk back down the tree and enable the child sections //TODO there are probably more efficient ways of doing this, but this will work for now
-				childPosIterator = quadNode.getChildPosIterator();
-				while (childPosIterator.hasNext())
+				for (int i = 0; i < 4; i++)
 				{
-					long childPos = childPosIterator.nextLong();
-					QuadNode<LodRenderSection> childNode = rootNode.getNode(childPos);
-					
-					boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading);
+					QuadNode<LodRenderSection> childNode = quadNode.getChildByIndex(i);
+					boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, DhSectionPos.getChildByIndex(sectionPos, i), parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading);
 					allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded;
 				}
 				if (!allChildrenSectionsAreLoaded)

core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java

@@ -73,8 +73,6 @@ public class RenderBufferHandler implements AutoCloseable
 	
 	private final AtomicBoolean rebuildAllBuffers = new AtomicBoolean(false);
 	
-	public F3Screen.MultiDynamicMessage f3Message;
-	
 	private int visibleBufferCount;
 	private int culledBufferCount;
 	private int shadowVisibleBufferCount;
@@ -104,31 +102,7 @@ public class RenderBufferHandler implements AutoCloseable
 		}
 		
 		
-		this.f3Message = new F3Screen.MultiDynamicMessage(
-			() ->
-			{
-				String countText = this.visibleBufferCount + "";
-				if (!Config.Client.Advanced.Graphics.AdvancedGraphics.disableFrustumCulling.get())
-				{
-					countText += "/" + (this.visibleBufferCount + this.culledBufferCount);
-				}
-				return LodUtil.formatLog("Rendered Buffer Count: " + countText);
-			}, 
-			() -> 
-			{
-				boolean hasIrisShaders = (IRIS_ACCESSOR != null && IRIS_ACCESSOR.isShaderPackInUse());
-				if (!hasIrisShaders)
-				{
-					return null;
-				}
-				
-				String countText = this.shadowVisibleBufferCount + "";
-				if (!Config.Client.Advanced.Graphics.AdvancedGraphics.disableFrustumCulling.get())
-				{
-					countText += "/" + (this.shadowVisibleBufferCount + this.shadowCulledBufferCount);
-				}
-				return LodUtil.formatLog("Shadow Buffer Count: " + countText);
-		});
+		F3Screen.setRenderBufferHandler(this);
 	}
 	
 	
@@ -396,6 +370,37 @@ public class RenderBufferHandler implements AutoCloseable
 	
 	
 	
+	//=========//
+	// F3 menu //
+	//=========//
+	
+	public String getVboRenderDebugMenuString()
+	{
+		String countText = F3Screen.NUMBER_FORMAT.format(this.visibleBufferCount);
+		if (!Config.Client.Advanced.Graphics.AdvancedGraphics.disableFrustumCulling.get())
+		{
+			countText += "/" + F3Screen.NUMBER_FORMAT.format(this.visibleBufferCount + this.culledBufferCount);
+		}
+		return LodUtil.formatLog("VBO Render Count: " + countText);
+	}
+	public String getShadowPassRenderDebugMenuString()
+	{
+		boolean hasIrisShaders = (IRIS_ACCESSOR != null && IRIS_ACCESSOR.isShaderPackInUse());
+		if (!hasIrisShaders)
+		{
+			return null;
+		}
+		
+		String countText = F3Screen.NUMBER_FORMAT.format(this.shadowVisibleBufferCount);
+		if (!Config.Client.Advanced.Graphics.AdvancedGraphics.disableFrustumCulling.get())
+		{
+			countText += "/" + F3Screen.NUMBER_FORMAT.format(this.shadowVisibleBufferCount + this.shadowCulledBufferCount);
+		}
+		return LodUtil.formatLog("Shadow VBO Render Count: " + countText);
+	}
+	
+	
+	
 	//=========//
 	// cleanup //
 	//=========//
@@ -413,7 +418,7 @@ public class RenderBufferHandler implements AutoCloseable
 			}
 		}
 		
-		this.f3Message.close();
+		F3Screen.setRenderBufferHandler(null);
 	}
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/sql/dto/ChunkHashDTO.java

@@ -0,0 +1,72 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020-2023 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.seibel.distanthorizons.core.sql.dto;
+
+import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
+import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
+import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
+import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
+import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import com.seibel.distanthorizons.core.pos.DhChunkPos;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import com.seibel.distanthorizons.core.util.FullDataPointUtil;
+import com.seibel.distanthorizons.core.util.LodUtil;
+import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
+import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
+import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
+import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.zip.Adler32;
+import java.util.zip.CheckedOutputStream;
+
+/** handles storing {@link FullDataSourceV2}'s in the database. */
+public class ChunkHashDTO implements IBaseDTO<DhChunkPos>
+{
+	public DhChunkPos pos;
+	public int chunkHash;
+	
+	
+	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public ChunkHashDTO(DhChunkPos pos, int chunkHash)
+	{
+		this.pos = pos;
+		this.chunkHash = chunkHash;
+	}
+	
+	
+	
+	//===========//
+	// overrides //
+	//===========//
+	
+	@Override 
+	public DhChunkPos getKey() { return this.pos; }
+	
+}

core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java

@@ -22,6 +22,7 @@ package com.seibel.distanthorizons.core.sql.dto;
 import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
 import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
 
 import java.io.ByteArrayInputStream;
@@ -81,6 +82,8 @@ public class FullDataSourceV1DTO implements IBaseDTO<Long>
 	
 	@Override
 	public Long getKey() { return this.pos; }
+	@Override
+	public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
 	
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java

@@ -25,6 +25,7 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
 import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
 import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.network.protocol.INetworkObject;
 import com.seibel.distanthorizons.core.util.FullDataPointUtil;
 import com.seibel.distanthorizons.core.util.LodUtil;
@@ -416,6 +417,8 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
 	
 	@Override
 	public Long getKey() { return this.pos; }
+	@Override
+	public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
 	
 	@Override
 	public String toString()

core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java

@@ -26,6 +26,8 @@ package com.seibel.distanthorizons.core.sql.dto;
 public interface IBaseDTO<TKey>
 {
 	TKey getKey();
+	/** Can be used for keys that don't have a clean human readable toString() method. */
+	default String getKeyDisplayString() { return this.getKey().toString(); }
 	
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java

@@ -173,7 +173,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
 		}
 		catch (DbConnectionClosedException ignored) 
 		{
-			LOGGER.warn("Attempted to insert [" + this.dtoClass.getSimpleName() + "] with primary key [" + (dto != null ? dto.getKey() : "NULL") + "] on closed repo [" + this.connectionString + "].");
+			LOGGER.warn("Attempted to insert ["+this.dtoClass.getSimpleName()+"] with primary key ["+(dto != null ? dto.getKeyDisplayString() : "NULL")+"] on closed repo ["+this.connectionString+"].");
 		}
 		catch (SQLException e)
 		{
@@ -190,7 +190,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
 		}
 		catch (DbConnectionClosedException e)
 		{
-			LOGGER.warn("Attempted to update [" + this.dtoClass.getSimpleName() + "] with primary key [" + (dto != null ? dto.getKey() : "NULL") + "] on closed repo [" + this.connectionString + "].");
+			LOGGER.warn("Attempted to update ["+this.dtoClass.getSimpleName()+"] with primary key ["+(dto != null ? dto.getKeyDisplayString() : "NULL")+"] on closed repo ["+this.connectionString+"].");
 		}
 		catch (SQLException e)
 		{

core/src/main/java/com/seibel/distanthorizons/core/sql/repo/ChunkHashRepo.java

@@ -0,0 +1,129 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020-2023 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.seibel.distanthorizons.core.sql.repo;
+
+import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
+import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.pos.DhChunkPos;
+import com.seibel.distanthorizons.core.sql.dto.ChunkHashDTO;
+import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
+import org.apache.logging.log4j.Logger;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Map;
+
+public class ChunkHashRepo extends AbstractDhRepo<DhChunkPos, ChunkHashDTO>
+{
+	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
+	
+	
+	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public ChunkHashRepo(String databaseType, String databaseLocation) throws SQLException
+	{
+		super(databaseType, databaseLocation, ChunkHashDTO.class);
+	}
+	
+	
+	
+	//===========//
+	// overrides //
+	//===========//
+	
+	@Override 
+	public String getTableName() { return "ChunkHash"; }
+	
+	@Override
+	public String createWhereStatement(DhChunkPos pos) { return "ChunkPosX = '"+pos.x+"' AND ChunkPosZ = '"+pos.z+"'"; }
+	
+	
+	
+	//=======================//
+	// repo required methods //
+	//=======================//
+	
+	@Override 
+	public ChunkHashDTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
+	{
+		int posX = (Integer) objectMap.get("ChunkPosX");
+		int posZ = (Integer) objectMap.get("ChunkPosZ");
+		
+		int chunkHash = (Integer) objectMap.get("ChunkHash");
+		
+		
+		ChunkHashDTO dto = new ChunkHashDTO(new DhChunkPos(posX, posZ), chunkHash);
+		return dto;
+	}
+	
+	@Override
+	public PreparedStatement createInsertStatement(ChunkHashDTO dto) throws SQLException
+	{
+		String sql =
+			"INSERT INTO "+this.getTableName() + " (\n" +
+			"   ChunkPosX, ChunkPosZ, \n" +
+			"   ChunkHash, \n" +
+			"   LastModifiedUnixDateTime, CreatedUnixDateTime) \n" +
+			"VALUES( \n" +
+			"    ?, ?, \n" +
+			"    ?, \n" +
+			"    ?, ? \n" +
+			");";
+		PreparedStatement statement = this.createPreparedStatement(sql);
+		
+		int i = 1;
+		statement.setObject(i++, dto.pos.x);
+		statement.setObject(i++, dto.pos.z);
+		
+		statement.setObject(i++, dto.chunkHash);
+		
+		statement.setObject(i++, System.currentTimeMillis()); // last modified unix time
+		statement.setObject(i++, System.currentTimeMillis()); // created unix time
+		
+		return statement;
+	}
+	
+	@Override
+	public PreparedStatement createUpdateStatement(ChunkHashDTO dto) throws SQLException
+	{
+		String sql =
+			"UPDATE "+this.getTableName()+" \n" +
+			"SET \n" +
+			"    ChunkHash = ? \n" +
+			"   ,LastModifiedUnixDateTime = ? \n" +
+			"WHERE ChunkPosX = ? AND ChunkPosZ = ?";
+		PreparedStatement statement = this.createPreparedStatement(sql);
+		
+		int i = 1;
+		statement.setObject(i++, dto.chunkHash);
+		statement.setObject(i++, System.currentTimeMillis()); // last modified unix time
+		
+		statement.setObject(i++, dto.pos.x);
+		statement.setObject(i++, dto.pos.z);
+		
+		return statement;
+	}
+	
+	
+	
+}

core/src/main/java/com/seibel/distanthorizons/core/util/gridList/ArrayGridList.java

@@ -30,6 +30,13 @@ public class ArrayGridList<T> extends ArrayList<T>
 {
 	public final int gridSize;
 	
+	
+	
+	//==============//
+	// constructors //
+	//==============//
+	
+	/** @param filler the function called for each index to set the initial values */
 	public ArrayGridList(int gridSize, BiFunction<Integer, Integer, T> filler)
 	{
 		super((gridSize) * (gridSize));
@@ -67,6 +74,12 @@ public class ArrayGridList<T> extends ArrayList<T>
 		// "==========================================\n");
 	}
 	
+	
+	
+	//=========//
+	// methods //
+	//=========//
+	
 	protected int _indexOf(int x, int y)
 	{
 		return x + y * gridSize;

core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/iterators/QuadNodeChildIndexIterator.java

@@ -22,15 +22,12 @@ package com.seibel.distanthorizons.core.util.objects.quadTree.iterators;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode;
 
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.NoSuchElementException;
-import java.util.Queue;
+import java.util.*;
 import java.util.function.Consumer;
 
 public class QuadNodeChildIndexIterator<T> implements Iterator<Integer>
 {
-	private final Queue<Integer> iteratorQueue = new LinkedList<>();
+	private final Queue<Integer> iteratorQueue = new ArrayDeque<>();
 	
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/iterators/QuadTreeNodeIterator.java

@@ -22,10 +22,7 @@ package com.seibel.distanthorizons.core.util.objects.quadTree.iterators;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode;
 
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.NoSuchElementException;
-import java.util.Queue;
+import java.util.*;
 import java.util.function.Consumer;
 
 public class QuadTreeNodeIterator<T> implements Iterator<QuadNode<T>>
@@ -34,8 +31,8 @@ public class QuadTreeNodeIterator<T> implements Iterator<QuadNode<T>>
 	private final byte highestDetailLevel;
 	
 	
-	private final Queue<QuadNode<T>> validNodesForDetailLevel = new LinkedList<>();
-	private final Queue<QuadNode<T>> iteratorNodeQueue = new LinkedList<>();
+	private final Queue<QuadNode<T>> validNodesForDetailLevel = new ArrayDeque<>();
+	private final Queue<QuadNode<T>> iteratorNodeQueue = new ArrayDeque<>();
 	private byte iteratorDetailLevel = 0;
 	
 	private final boolean onlyReturnLeafValues;
@@ -64,7 +61,7 @@ public class QuadTreeNodeIterator<T> implements Iterator<QuadNode<T>>
 			// but it is simple and functions well enough for now
 			
 			
-			Queue<QuadNode<T>> parentNodeQueue = new LinkedList<>();
+			Queue<QuadNode<T>> parentNodeQueue = new ArrayDeque<>();
 			parentNodeQueue.add(rootNode);
 			
 			// walk through the whole tree and add each leaf node to the iterator queue

core/src/main/java/com/seibel/distanthorizons/core/world/AbstractDhWorld.java

@@ -20,6 +20,7 @@
 package com.seibel.distanthorizons.core.world;
 
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.logging.f3.F3Screen;
 import org.apache.logging.log4j.Logger;
 
 import java.io.Closeable;
@@ -36,11 +37,22 @@ public abstract class AbstractDhWorld implements IDhWorld, Closeable
 	
 	
 	
+	// constructor //
+	
 	protected AbstractDhWorld(EWorldEnvironment environment) { this.environment = environment; }
 	
 	
-	// remove the "throws IOException"
+	
+	// abstract methods //
+	
+	// removes the "throws IOException"
 	@Override
 	public abstract void close();
 	
+	
+	
+	// helper methods //
+	
+	public String GetDebugMenuString() { return this.environment + " World with " + F3Screen.NUMBER_FORMAT.format(this.getLoadedLevelCount()) + " levels"; }
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java

@@ -46,8 +46,6 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor
 	public ExecutorService dhTickerThread = ThreadUtil.makeSingleThreadPool("Client Server World Ticker Thread", 2);
 	public EventLoop eventLoop = new EventLoop(this.dhTickerThread, this::_clientTick); //TODO: Rate-limit the loop
 	
-	public F3Screen.DynamicMessage f3Message;
-	
 	
 	
 	//=============//
@@ -57,10 +55,7 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor
 	public DhClientServerWorld()
 	{
 		super(EWorldEnvironment.Client_Server);
-		
 		LOGGER.info("Started DhWorld of type " + this.environment);
-		
-		this.f3Message = new F3Screen.DynamicMessage(() -> LodUtil.formatLog(this.environment + " World with " + this.dhLevels.size() + " levels"));
 	}
 	
 	
@@ -113,6 +108,8 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor
 	
 	@Override
 	public Iterable<? extends IDhLevel> getAllLoadedLevels() { return this.dhLevels; }
+	@Override 
+	public int getLoadedLevelCount() { return this.dhLevels.size(); }
 	
 	@Override
 	public void unloadLevel(@NotNull ILevelWrapper wrapper)
@@ -154,13 +151,16 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor
 	
 	public void doWorldGen() { this.dhLevels.forEach(DhClientServerLevel::doWorldGen); }
 	
+	
+	
+	//================//
+	// base overrides //
+	//================//
+	
 	/** synchronized to prevent a rare issue where the server tries closing the same world multiple times in rapid succession. */
 	@Override
 	public synchronized void close()
 	{
-		this.f3Message.close();
-		
-		
 		// clear dhLevels to prevent concurrent modification errors
 		HashSet<DhClientServerLevel> levelsToClose = new HashSet<>(this.dhLevels);
 		this.dhLevels.clear();

core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java

@@ -99,6 +99,8 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld
 	
 	@Override
 	public Iterable<? extends IDhLevel> getAllLoadedLevels() { return this.levels.values(); }
+	@Override
+	public int getLoadedLevelCount() { return this.levels.size(); }
 	
 	@Override
 	public void unloadLevel(@NotNull ILevelWrapper wrapper)

core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java

@@ -119,6 +119,8 @@ public class DhServerWorld extends AbstractDhWorld implements IDhServerWorld
 	
 	@Override
 	public Iterable<? extends IDhLevel> getAllLoadedLevels() { return this.levels.values(); }
+	@Override
+	public int getLoadedLevelCount() { return this.levels.size(); }
 	
 	@Override
 	public void unloadLevel(@NotNull ILevelWrapper wrapper)
@@ -146,6 +148,12 @@ public class DhServerWorld extends AbstractDhWorld implements IDhServerWorld
 		this.levels.values().forEach(DhServerLevel::doWorldGen);
 	}
 	
+	
+	
+	//================//
+	// base overrides //
+	//================//
+	
 	@Override
 	public void close()
 	{

core/src/main/java/com/seibel/distanthorizons/core/world/IDhWorld.java

@@ -31,6 +31,7 @@ public interface IDhWorld
 	IDhLevel getOrLoadLevel(@NotNull ILevelWrapper levelWrapper);
 	IDhLevel getLevel(@NotNull ILevelWrapper wrapper);
 	Iterable<? extends IDhLevel> getAllLoadedLevels();
+	int getLoadedLevelCount();
 	
 	void unloadLevel(@NotNull ILevelWrapper levelWrapper);
 	

core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/chunk/ChunkLightStorage.java

@@ -0,0 +1,257 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020-2023 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.seibel.distanthorizons.core.wrapperInterfaces.chunk;
+
+import com.seibel.distanthorizons.core.util.LodUtil;
+import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/**
+ * Compact, efficient storage for light levels. 
+ * all blocks only take up 4 bits in total, 
+ * and if a 16x16x16 area is detected to have the same light level in all positions,
+ * then we store a single byte for that light level, instead of 2 kilobytes.
+ * 
+ * @author Builderb0y
+*/
+public class ChunkLightStorage 
+{
+	/** the minimum Y level in the chunk which this storage is storing light levels for (inclusive). */
+	public int minY;
+	/** the maximum Y level in the chunk which this storage is storing light levels for (exclusive). */
+	public int maxY;
+	
+	/** the data stored in this storage, split up into 16x16x16 areas. */
+	public LightSection[] lightSections;
+	
+	/** 
+	 * If the get method is called on a Y position above what's stored
+	 * this value will be returned. <br><br>
+	 * 
+	 * This needs to be manually defined since sky and block lights behave differently
+	 * for values both above and below what's defined.
+	 */
+	public int aboveMaxYValue;
+	/** @see ChunkLightStorage#aboveMaxYValue */
+	public int belowMinYValue;
+	
+	
+	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public static ChunkLightStorage createSkyLightStorage(IChunkWrapper chunkWrapper) { return createSkyLightStorage(chunkWrapper.getMinBuildHeight(), chunkWrapper.getMaxBuildHeight());  }
+	public static ChunkLightStorage createSkyLightStorage(int minY, int maxY)
+	{
+		return new ChunkLightStorage(
+				minY, maxY,
+				// positions above should be lit but positions below should be unlit
+				LodUtil.MAX_MC_LIGHT, LodUtil.MIN_MC_LIGHT);
+	}
+	public static ChunkLightStorage createBlockLightStorage(IChunkWrapper chunkWrapper) { return createBlockLightStorage(chunkWrapper.getMinBuildHeight(), chunkWrapper.getMaxBuildHeight());  }
+	public static ChunkLightStorage createBlockLightStorage(int minY, int maxY)
+	{
+		return new ChunkLightStorage(
+				minY, maxY,
+				// positions above and below the handled area should be unlit
+				LodUtil.MIN_MC_LIGHT, LodUtil.MIN_MC_LIGHT);
+	}
+	
+	public ChunkLightStorage(int minY, int maxY, int aboveMaxYValue, int belowMinYValue) 
+	{
+		this.minY = minY;
+		this.maxY = maxY;
+		
+		this.aboveMaxYValue = aboveMaxYValue;
+		this.belowMinYValue = belowMinYValue;
+	}
+	
+	
+	
+	//=====================//
+	// getters and setters //
+	//=====================//
+	
+	public int get(int x, int y, int z)
+	{
+		if (y < this.minY)
+		{
+			return this.belowMinYValue;
+		}
+		else if (y >= this.maxY)
+		{
+			return  this.aboveMaxYValue;
+		}
+		
+		
+		if (this.lightSections != null)
+		{
+			LightSection lightSection = this.lightSections[BitShiftUtil.divideByPowerOfTwo(y - this.minY, 4)];
+			if (lightSection != null)
+			{
+				return lightSection.get(x, y, z);
+			}
+		}
+		
+		return 0;
+	}
+	
+	public void set(int x, int y, int z, int lightLevel)
+	{
+		if (y < this.minY || y >= this.maxY)
+		{
+			return;
+		}
+		
+		//populate array if it doesn't exist.
+		if (this.lightSections == null)
+		{
+			this.lightSections = new LightSection[BitShiftUtil.divideByPowerOfTwo(this.maxY - this.minY, 4)];
+		}
+		
+		int index = (y - this.minY) >> 4;
+		LightSection lightSection = this.lightSections[index];
+		
+		//populate lightSection in array if it doesn't exist.
+		if (lightSection == null)
+		{
+			lightSection = new LightSection(0);
+			this.lightSections[index] = lightSection;
+		}
+		lightSection.set(x, y, z, lightLevel);
+	}
+	
+	
+	
+	//================//
+	// helper classes //
+	//================//
+	
+	public static class LightSection
+	{
+		public byte constantValue;
+		public long[] data;
+		public short[] counts;
+		
+		public LightSection(int initialValue)
+		{
+			this.constantValue = (byte) (initialValue);
+			this.counts = new short[16];
+			this.counts[initialValue] = 16 * 16 * 16;
+		}
+		
+		public int get(int x, int y, int z)
+		{
+			if (this.constantValue >= 0)
+			{
+				return this.constantValue;
+			}
+			
+			x &= 15;
+			y &= 15;
+			z &= 15;
+			long bits = this.data[(z << 4) | x];
+			return ((int) (bits >>> (y << 2))) & 15;
+		}
+		
+		public void set(int x, int y, int z, int lightLevel)
+		{
+			int oldLightLevel = -1;
+			if (this.constantValue >= 0)
+			{
+				oldLightLevel = this.constantValue;
+				
+				//if the light level didn't change, then there's nothing to do.
+				if (oldLightLevel == lightLevel) return;
+				
+				//if we are a constant value and need to change something,
+				//then that means we need to convert to a non-constant value.
+				this.data = DataRecycler.get();
+				
+				//repeat oldLightLevel 16 times as a bit pattern.
+				long payload = oldLightLevel;
+				payload |= payload << 4;
+				payload |= payload << 8;
+				payload |= payload << 16;
+				payload |= payload << 32;
+				
+				//fill our data with our constant value.
+				Arrays.fill(this.data, payload);
+				
+				//we are no longer a constant value.
+				this.constantValue = -1;
+			}
+			
+			x &= 15;
+			y &= 15;
+			z &= 15;
+			int index = (z << 4) | x;
+			long bits = this.data[index];
+			//if we weren't a constant value before, now's the time to initialize oldLightLevel.
+			if (oldLightLevel < 0)
+			{
+				oldLightLevel = ((int) (bits >>> (y << 2))) & 15;
+			}
+			//clear the 4 bits that correspond to the light level at x, y, z...
+			bits &= ~(15L << (y << 2));
+			//...and then re-populate those bits with the new light level.
+			bits |= ((long) (lightLevel)) << (y << 2);
+			//store the updated bits in our data.
+			this.data[index] = bits;
+			
+			//we have one less of the old light level...
+			this.counts[oldLightLevel]--;
+			//...and one more of the new level.
+			//if the number associated with the new level is now 4096 (AKA 16 ^ 3),
+			//then this implies every position in this section has the same light level,
+			//and therefore we can convert back to a constant value.
+			if (++this.counts[lightLevel] == 4096)
+			{
+				this.constantValue = (byte) (lightLevel);
+				DataRecycler.reclaim(this.data);
+				this.data = null;
+			}
+		}
+		
+	}
+	
+	static class DataRecycler
+	{
+		private static final ArrayList<long[]> recycled = new ArrayList<>(256);
+		
+		static synchronized long[] get()
+		{
+			if (recycled.isEmpty())
+			{
+				return new long[256];
+			}
+			else
+			{
+				return recycled.remove(recycled.size() - 1);
+			}
+		}
+		
+		static synchronized void reclaim(long[] data) { if (recycled.size() < 256) recycled.add(data); }
+	}
+	
+}

core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/chunk/IChunkWrapper.java

@@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.pos.DhBlockPos;
 import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.pos.DhChunkPos;
 import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
+import com.seibel.distanthorizons.coreapi.ModInfo;
 import com.seibel.distanthorizons.coreapi.interfaces.dependencyInjection.IBindable;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
@@ -31,6 +32,11 @@ import java.util.ArrayList;
 
 public interface IChunkWrapper extends IBindable
 {
+	/** useful for debugging, but can slow down chunk operations quite a bit due to being called every time. */
+	boolean RUN_RELATIVE_POS_INDEX_VALIDATION = ModInfo.IS_DEV_BUILD;
+	
+	
+	
 	DhChunkPos getChunkPos();
 	
 	default int getHeight() { return this.getMaxBuildHeight() - this.getMinBuildHeight(); }
@@ -61,8 +67,6 @@ public interface IChunkWrapper extends IBindable
 	int getMinBlockX();
 	int getMinBlockZ();
 	
-	long getLongChunkPos();
-	
 	void setIsDhLightCorrect(boolean isDhLightCorrect);
 	void setUseDhLighting(boolean useDhLighting);
 	boolean isLightCorrect();
@@ -77,6 +81,65 @@ public interface IChunkWrapper extends IBindable
 	int getBlockLight(int relX, int relY, int relZ);
 	int getSkyLight(int relX, int relY, int relZ);
 	
+	
+	ArrayList<DhBlockPos> getBlockLightPosList();
+	
+	
+	default boolean blockPosInsideChunk(DhBlockPos blockPos) { return this.blockPosInsideChunk(blockPos.x, blockPos.y, blockPos.z); }
+	default boolean blockPosInsideChunk(int x, int y, int z)
+	{
+		return (x >= this.getMinBlockX() && x <= this.getMaxBlockX()
+				&& y >= this.getMinBuildHeight() && y < this.getMaxBuildHeight()
+				&& z >= this.getMinBlockZ() && z <= this.getMaxBlockZ());
+	}
+	default boolean blockPosInsideChunk(DhBlockPos2D blockPos)
+	{
+		return (blockPos.x >= this.getMinBlockX() && blockPos.x <= this.getMaxBlockX()
+				&& blockPos.z >= this.getMinBlockZ() && blockPos.z <= this.getMaxBlockZ());
+	}
+	
+	boolean doNearbyChunksExist();
+	String toString();
+	
+	
+	default IBlockStateWrapper getBlockState(DhBlockPos pos) { return this.getBlockState(pos.x, pos.y, pos.z); }
+	IBlockStateWrapper getBlockState(int relX, int relY, int relZ);
+	
+	IBiomeWrapper getBiome(int relX, int relY, int relZ);
+	
+	boolean isStillValid();
+	
+	
+	
+	//========================//
+	// default helper methods //
+	//========================//
+	
+	/** used to prevent accidentally attempting to get/set values outside this chunk's boundaries */
+	default void throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(int x, int y, int z) throws IndexOutOfBoundsException
+	{
+		if (!RUN_RELATIVE_POS_INDEX_VALIDATION)
+		{
+			return;
+		}
+		
+		
+		// FIXME +1 is to handle the fact that LodDataBuilder adds +1 to all block lighting calculations, also done in the constructor
+		int minHeight = this.getMinBuildHeight();
+		int maxHeight = this.getMaxBuildHeight() + 1;
+		
+		if (x < 0 || x >= LodUtil.CHUNK_WIDTH
+				|| z < 0 || z >= LodUtil.CHUNK_WIDTH
+				|| y < minHeight || y > maxHeight)
+		{
+			String errorMessage = "Relative position [" + x + "," + y + "," + z + "] out of bounds. \n" +
+					"X/Z must be between 0 and 15 (inclusive) \n" +
+					"Y must be between [" + minHeight + "] and [" + maxHeight + "] (inclusive).";
+			throw new IndexOutOfBoundsException(errorMessage);
+		}
+	}
+	
+	
 	/**
 	 * Populates DH's saved lighting using MC's lighting engine.
 	 * This is generally done in cases where MC's lighting is correct now, but may not be later (like when a chunk is unloading).
@@ -108,27 +171,38 @@ public interface IChunkWrapper extends IBindable
 	}
 	
 	
-	
-	ArrayList<DhBlockPos> getBlockLightPosList();
-	
-	
-	default boolean blockPosInsideChunk(DhBlockPos blockPos) { return this.blockPosInsideChunk(blockPos.x, blockPos.y, blockPos.z); }
-	default boolean blockPosInsideChunk(int x, int y, int z)
+	/**
+	 * Converts a 3D position into a 1D array index. <br><br>
+	 *
+	 * Source: <br>
+	 * <a href="https://stackoverflow.com/questions/7367770/how-to-flatten-or-index-3d-array-in-1d-array">stackoverflow</a>
+	 */
+	default int relativeBlockPosToIndex(int xRel, int y, int zRel)
 	{
-		return (x >= this.getMinBlockX() && x <= this.getMaxBlockX()
-				&& y >= this.getMinBuildHeight() && y < this.getMaxBuildHeight()
-				&& z >= this.getMinBlockZ() && z <= this.getMaxBlockZ());
-	}
-	default boolean blockPosInsideChunk(DhBlockPos2D blockPos)
-	{
-		return (blockPos.x >= this.getMinBlockX() && blockPos.x <= this.getMaxBlockX()
-				&& blockPos.z >= this.getMinBlockZ() && blockPos.z <= this.getMaxBlockZ());
+		int yRel = y - this.getMinBuildHeight();
+		return (zRel * LodUtil.CHUNK_WIDTH * this.getHeight()) + (yRel * LodUtil.CHUNK_WIDTH) + xRel;
 	}
 	
-	boolean doNearbyChunksExist();
-	String toString();
+	/**
+	 * Converts a 3D position into a 1D array index. <br><br>
+	 *
+	 * Source: <br>
+	 * <a href="https://stackoverflow.com/questions/7367770/how-to-flatten-or-index-3d-array-in-1d-array">stackoverflow</a>
+	 */
+	default DhBlockPos indexToRelativeBlockPos(int index)
+	{
+		final int zRel = index / (LodUtil.CHUNK_WIDTH * this.getHeight());
+		index -= (zRel * LodUtil.CHUNK_WIDTH * this.getHeight());
+		
+		final int y = index / LodUtil.CHUNK_WIDTH;
+		final int yRel = y + this.getMinBuildHeight();
+		
+		final int xRel = index % LodUtil.CHUNK_WIDTH;
+		return new DhBlockPos(xRel, yRel, zRel);
+	}
 	
-	/** This is a bad hash algorithm, but can be used for rough debugging. */
+	
+	/** This is a bad hash algorithm since it only uses the heightmap, but can be used for rough debugging. */
 	default int roughHashCode()
 	{
 		int hash = 31;
@@ -138,7 +212,31 @@ public interface IChunkWrapper extends IBindable
 		{
 			for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++)
 			{
-				hash = hash * primeMultiplier + Integer.hashCode(this.getLightBlockingHeightMapValue(x, z));
+				hash = (hash * primeMultiplier) + Integer.hashCode(this.getLightBlockingHeightMapValue(x, z));
+			}
+		}
+		
+		return hash;
+	}
+	
+	default int getBlockBiomeHashCode()
+	{
+		int hash = 31;
+		int primeBlockMultiplier = 227;
+		int primeBiomeMultiplier = 701;
+		
+		int minBuildHeight = this.getMinBuildHeight();
+		int maxBuildHeight = this.getMaxBuildHeight();
+		
+		for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++)
+		{
+			for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++)
+			{
+				for (int y = minBuildHeight; y < maxBuildHeight; y++)
+				{
+					hash = (hash * primeBlockMultiplier) + this.getBlockState(x, y, z).hashCode();
+					hash = (hash * primeBiomeMultiplier) + this.getBiome(x, y, z).hashCode();
+				}
 			}
 		}
 		
@@ -146,11 +244,4 @@ public interface IChunkWrapper extends IBindable
 	}
 	
 	
-	default IBlockStateWrapper getBlockState(DhBlockPos pos) { return this.getBlockState(pos.x, pos.y, pos.z); }
-	IBlockStateWrapper getBlockState(int relX, int relY, int relZ);
-	
-	IBiomeWrapper getBiome(int relX, int relY, int relZ);
-	
-	boolean isStillValid();
-	
 }

core/src/main/resources/assets/distanthorizons/lang/en_us.json

@@ -423,7 +423,7 @@
   "distanthorizons.config.client.advanced.multiThreading.numberOfFileHandlerThreads":
     "NO. of file handler threads",
   "distanthorizons.config.client.advanced.multiThreading.numberOfFileHandlerThreads.@tooltip":
-    "The number of threads used when building vertex buffers \n(The things sent to your GPU to draw the LODs). \nCan only be between 1 and your CPU's processor count.",
+    "The number of threads used when reading/writing LOD data to/from the disk. \nCan only be between 1 and your CPU's processor count.",
   "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForFileHandlerThreads":
     "Runtime % for file handler threads",
   
@@ -545,6 +545,8 @@
     "File Sub Dimension Events",
   "distanthorizons.config.client.advanced.logging.logNetworkEvent":
     "Network Events",
+  "distanthorizons.config.client.advanced.logging.showLowMemoryWarningOnStartup":
+    "Show Low Memory Warning",
     
     
     

core/src/main/resources/sqlScripts/0060-sqlite-createChunkHashTable.sql

@@ -0,0 +1,13 @@
+
+CREATE TABLE ChunkHash(
+    -- compound primary key
+     ChunkPosX INT NOT NULL
+    ,ChunkPosZ INT NOT NULL
+    
+    ,ChunkHash INT NOT NULL
+    
+    ,LastModifiedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    ,CreatedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    
+    ,PRIMARY KEY (ChunkPosX, ChunkPosZ)
+);

core/src/main/resources/sqlScripts/scriptList.txt

@@ -5,3 +5,4 @@
 0031-sqlite-useSqliteWalJournaling.sql
 0040-sqlite-removeRenderCache.sql
 0050-sqlite-addApplyToParentIndex.sql
+0060-sqlite-createChunkHashTable.sql

core/src/test/java/testItems/lightingEngine/LightingTestBlockStateWrapper.java

@@ -0,0 +1,107 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020-2023 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package testItems.lightingEngine;
+
+import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
+import tests.LightingEngineTest;
+
+/**
+ * @see LightingEngineTest
+ * @see LightingTestChunkWrapper
+ */
+public class LightingTestBlockStateWrapper implements IBlockStateWrapper
+{
+	private int opacity = -1;
+	private int lightEmission = -1;
+	
+	
+	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public LightingTestBlockStateWrapper(int opacity, int lightEmission)
+	{
+		this.opacity = opacity;
+		this.lightEmission = lightEmission;
+	}
+	
+	
+	
+	//=================//
+	// wrapper methods //
+	//=================//
+	
+	@Override
+	public int getOpacity() { return this.opacity; }
+	
+	@Override
+	public int getLightEmission() { return this.lightEmission; }
+	
+	
+	
+	
+	//===============//
+	// unimplemented //
+	//===============//
+	
+	//@Override
+	//public boolean equals(Object obj)
+	//{
+	//	if (this == obj)
+	//	{
+	//		return true;
+	//	}
+	//	
+	//	if (obj == null || this.getClass() != obj.getClass())
+	//	{
+	//		return false;
+	//	}
+	//	
+	//	BlockStateTestWrapper that = (BlockStateTestWrapper) obj;
+	//	// the serialized value is used so we can test the contents instead of the references
+	//	return Objects.equals(this.getSerialString(), that.getSerialString());
+	//}
+	
+	//@Override
+	//public int hashCode() { return this.hashCode; }
+	//@Override
+	//public String toString() { return this.getSerialString(); }
+	
+	
+	@Override
+	public String getSerialString() { throw new UnsupportedOperationException("Not Implemented"); }
+	
+	@Override
+	public Object getWrappedMcObject() { throw new UnsupportedOperationException("Not Implemented"); }
+	
+	@Override
+	public boolean isAir() { throw new UnsupportedOperationException("Not Implemented"); }
+	
+	@Override
+	public boolean isSolid() { throw new UnsupportedOperationException("Not Implemented"); }
+	
+	@Override
+	public boolean isLiquid() { throw new UnsupportedOperationException("Not Implemented"); }
+	
+	@Override
+	public byte getIrisBlockMaterialId() { throw new UnsupportedOperationException("Not Implemented"); }
+	
+}

core/src/test/java/testItems/lightingEngine/LightingTestChunkWrapper.java

@@ -0,0 +1,421 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020-2023 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package testItems.lightingEngine;
+
+import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.pos.DhBlockPos;
+import com.seibel.distanthorizons.core.pos.DhChunkPos;
+import com.seibel.distanthorizons.core.util.LodUtil;
+import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
+import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
+import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.ChunkLightStorage;
+import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
+import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
+import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+import org.apache.logging.log4j.Logger;
+import tests.LightingEngineTest;
+
+import java.io.*;
+import java.util.ArrayList;
+
+/** 
+ * @see LightingEngineTest
+ * @see LightingTestBlockStateWrapper
+ */
+public class LightingTestChunkWrapper implements IChunkWrapper
+{
+	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
+	
+	
+	// chunk values //
+	
+	private final DhChunkPos chunkPos;
+	private ChunkLightStorage blockLightStorage;
+	private ChunkLightStorage skyLightStorage;
+	
+	private ArrayList<DhBlockPos> blockLightPosList = null;
+	
+	private boolean useDhLighting;
+	
+	private int minNonEmptyHeight = Integer.MIN_VALUE;
+	private int maxNonEmptyHeight = Integer.MAX_VALUE;
+	
+	
+	// test values //
+	
+	private final Int2IntOpenHashMap blockOpacityStorage;
+	private final Int2IntOpenHashMap blockEmissionStorage;
+	private final int[][] solidHeightMap = new int[LodUtil.CHUNK_WIDTH][LodUtil.CHUNK_WIDTH];
+	private final int[][] lightBlockingHeightMap = new int[LodUtil.CHUNK_WIDTH][LodUtil.CHUNK_WIDTH];
+	
+	
+	
+	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public LightingTestChunkWrapper(IChunkWrapper chunkWrapper)
+	{
+		this.chunkPos = chunkWrapper.getChunkPos();
+		
+		this.blockOpacityStorage = new Int2IntOpenHashMap();
+		this.blockEmissionStorage = new Int2IntOpenHashMap();
+		this.blockLightPosList = new ArrayList<>();
+		
+		for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++)
+		{
+			for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++)
+			{
+				for (int y = this.getMinBuildHeight(); y < this.getMaxBuildHeight(); y++)
+				{
+					IBlockStateWrapper block = chunkWrapper.getBlockState(x,y,z);
+					
+					int opacity = block.getOpacity();
+					if (opacity >= IBlockStateWrapper.FULLY_OPAQUE)
+					{
+						opacity = 3;
+					}
+					
+					this.blockOpacityStorage.put(new DhBlockPos(x, y, z).hashCode(), opacity);
+					this.blockEmissionStorage.put(new DhBlockPos(x, y, z).hashCode(), block.getLightEmission());
+					
+					if (block.getLightEmission() != 0)
+					{
+						this.blockLightPosList.add(new DhBlockPos(x,y,z));
+					}
+				}
+				
+				this.lightBlockingHeightMap[x][z] = chunkWrapper.getLightBlockingHeightMapValue(x, z);
+				this.solidHeightMap[x][z] = chunkWrapper.getSolidHeightMapValue(x, z);
+				
+			}
+		}
+	}
+	
+	
+	
+	//===============//
+	// file handling //
+	//===============//
+	
+	public LightingTestChunkWrapper(DhChunkPos pos, File saveFile) throws DataCorruptedException
+	{
+		this.chunkPos = pos;
+		
+		this.blockOpacityStorage = new Int2IntOpenHashMap();
+		this.blockEmissionStorage = new Int2IntOpenHashMap();
+		this.blockLightPosList = new ArrayList<>();
+		
+		try(FileInputStream inputStream = new FileInputStream(saveFile);
+			BufferedInputStream bufferedStream = new BufferedInputStream(inputStream);
+			DataInputStream stream = new DataInputStream(bufferedStream))
+		{
+			for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++)
+			{
+				for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++)
+				{
+					for (int y = this.getMinBuildHeight(); y < this.getMaxBuildHeight(); y++)
+					{
+						this.blockOpacityStorage.put(new DhBlockPos(x, y, z).hashCode(), stream.readInt());
+						
+						int blockEmission = stream.readInt();
+						this.blockEmissionStorage.put(new DhBlockPos(x, y, z).hashCode(), blockEmission);
+						if (blockEmission != 0)
+						{
+							this.blockLightPosList.add(new DhBlockPos(x,y,z));
+						}
+					}
+					
+					if (stream.readChar() != ';')
+					{
+						throw new DataCorruptedException("bad height map");
+					}
+					
+					this.lightBlockingHeightMap[x][z] = stream.readInt();
+					this.solidHeightMap[x][z] = stream.readInt();
+					
+					if (stream.readChar() != '\n')
+					{
+						throw new DataCorruptedException(" bad col ending");
+					}
+				}
+			}
+		}
+		catch (IOException e)
+		{
+			LOGGER.error("Unable to write to file: ["+e.getMessage()+"].", e);
+		}
+	}
+	public void writeToFile(File file)
+	{
+		try(FileOutputStream fileStream = new FileOutputStream(file);
+			BufferedOutputStream bufferedStream = new BufferedOutputStream(fileStream);
+			DataOutputStream stream = new DataOutputStream(bufferedStream))
+		{
+			for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++)
+			{
+				for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++)
+				{
+					for (int y = this.getMinBuildHeight(); y < this.getMaxBuildHeight(); y++)
+					{
+						stream.writeInt(this.blockOpacityStorage.get(new DhBlockPos(x, y, z).hashCode()));
+						stream.writeInt(this.blockEmissionStorage.get(new DhBlockPos(x, y, z).hashCode()));
+					}
+					
+					stream.writeChar(';');
+					
+					stream.writeInt(this.lightBlockingHeightMap[x][z]);
+					stream.writeInt(this.solidHeightMap[x][z]);
+					
+					stream.writeChar('\n');
+				}
+			}
+			
+			stream.flush();
+		}
+		catch (IOException e)
+		{
+			LOGGER.error("Unable to write to file: ["+e.getMessage()+"].", e);
+		}
+	}
+	
+	/** 
+	 * Can be added into {@link com.seibel.distanthorizons.core.api.internal.SharedApi#applyChunkUpdate(IChunkWrapper, ILevelWrapper, boolean)} 
+	 * to save chunks to file for future testing.
+	 */
+	public void tryConvertingAndSavingChunkWrapper(IChunkWrapper chunkWrapper)
+	{
+		try
+		{
+			File chunkFile = new File(LightingEngineTest.TEST_DATA_PATH + "/" + chunkWrapper.getChunkPos().toString());
+			if (!chunkFile.exists())
+			{
+				LightingTestChunkWrapper testWrapper = new LightingTestChunkWrapper(chunkWrapper);
+				testWrapper.writeToFile(chunkFile);
+			}
+		}
+		catch (Exception e)
+		{
+			LOGGER.error(e.getMessage(), e);
+		}
+	}
+	
+	
+	
+	//===============//
+	// chunk methods //
+	//===============//
+	
+	@Override
+	public int getHeight() { return 255; }
+	
+	@Override
+	public int getMinBuildHeight() { return -64; }
+	@Override
+	public int getMaxBuildHeight() { return 255; }
+	
+	@Override
+	public int getMinNonEmptyHeight()
+	{
+		if (this.minNonEmptyHeight != Integer.MIN_VALUE)
+		{
+			return this.minNonEmptyHeight;
+		}
+		
+		
+		// default if every section is empty or missing
+		this.minNonEmptyHeight = this.getMinBuildHeight();
+		
+		// determine the lowest empty section (bottom up)
+		int maxYHeight = this.getMaxBuildHeight();
+		for (int y = this.getMinBuildHeight(); y < maxYHeight; y++)
+		{
+			if (this.blockOpacityStorage.get(new DhBlockPos(0, y, 0).hashCode()) != 0)
+			{
+				// -16 to simulate having to populate the full chunk section
+				this.minNonEmptyHeight = Math.min(y - 16, maxYHeight);
+				break;
+			}
+		}
+		
+		return this.minNonEmptyHeight;
+	}
+	
+	
+	@Override
+	public int getMaxNonEmptyHeight()
+	{
+		if (this.maxNonEmptyHeight != Integer.MAX_VALUE)
+		{
+			return this.maxNonEmptyHeight;
+		}
+		
+		
+		// default if every section is empty or missing
+		this.maxNonEmptyHeight = this.getMaxBuildHeight();
+		
+		// determine the highest empty section (top down)
+		int minYHeight = this.getMinBuildHeight();
+		for (int y = this.getMaxBuildHeight(); y >= minYHeight; y--)
+		{
+			if (this.blockOpacityStorage.get(new DhBlockPos(0, y, 0).hashCode()) != 0)
+			{
+				// -16 to simulate having to populate the full chunk section
+				this.maxNonEmptyHeight = Math.max(y - 16, minYHeight);
+				break;
+			}
+		}
+		
+		return this.maxNonEmptyHeight;
+	}
+	
+	
+	@Override
+	public int getSolidHeightMapValue(int xRel, int zRel) { return this.solidHeightMap[xRel][zRel]; }
+	
+	@Override
+	public int getLightBlockingHeightMapValue(int xRel, int zRel) { return this.lightBlockingHeightMap[xRel][zRel]; }
+	
+	
+	
+	@Override
+	public IBiomeWrapper getBiome(int relX, int relY, int relZ) { throw new UnsupportedOperationException("Not implemented"); }
+	
+	@Override
+	public DhChunkPos getChunkPos() { return this.chunkPos; }
+	
+	@Override
+	public int getMaxBlockX() { return 0; }
+	@Override
+	public int getMaxBlockZ() { return 0; }
+	@Override
+	public int getMinBlockX() { return LodUtil.CHUNK_WIDTH; }
+	@Override
+	public int getMinBlockZ() { return LodUtil.CHUNK_WIDTH; }
+	
+	@Override
+	public void setIsDhLightCorrect(boolean isDhLightCorrect) {  }
+	
+	@Override
+	public void setUseDhLighting(boolean useDhLighting) { this.useDhLighting = useDhLighting; }
+	
+	
+	
+	@Override
+	public boolean isLightCorrect() { return false; }
+	
+	
+	@Override
+	public int getDhBlockLight(int relX, int y, int relZ)
+	{
+		this.throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(relX, y, relZ);
+		return this.getBlockLightStorage().get(relX, y, relZ);
+	}
+	@Override
+	public void setDhBlockLight(int relX, int y, int relZ, int lightValue)
+	{
+		this.throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(relX, y, relZ);
+		this.getBlockLightStorage().set(relX, y, relZ, lightValue);
+	}
+	
+	private ChunkLightStorage getBlockLightStorage()
+	{
+		if (this.blockLightStorage == null)
+		{
+			this.blockLightStorage = new ChunkLightStorage(
+					// +/- 16 is to fix an issue with the test chunk where the storage isn't big enough,
+					// James probably just screwed up the min/max height slightly 
+					this.getMinBuildHeight() - 16, this.getMaxBuildHeight() + 16, 
+					// positions above and below the handled area should be unlit
+					LodUtil.MIN_MC_LIGHT, LodUtil.MIN_MC_LIGHT);
+		}
+		return this.blockLightStorage;
+	}
+	
+	
+	@Override
+	public int getDhSkyLight(int relX, int y, int relZ)
+	{
+		this.throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(relX, y, relZ);
+		return this.getSkyLightStorage().get(relX, y, relZ);
+	}
+	@Override
+	public void setDhSkyLight(int relX, int y, int relZ, int lightValue)
+	{
+		this.throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(relX, y, relZ);
+		this.getSkyLightStorage().set(relX, y, relZ, lightValue);
+	}
+	
+	private ChunkLightStorage getSkyLightStorage()
+	{
+		if (this.skyLightStorage == null)
+		{
+			this.skyLightStorage = new ChunkLightStorage(
+					// +/- 16 is to fix an issue with the test chunk where the storage isn't big enough,
+					// James probably just screwed up the min/max height slightly
+					this.getMinBuildHeight() - 16, this.getMaxBuildHeight() + 16,
+					// positions above should be lit but positions below should be unlit
+					LodUtil.MAX_MC_LIGHT, LodUtil.MIN_MC_LIGHT);
+		}
+		return this.skyLightStorage;
+	}
+	
+	
+	@Override
+	public int getBlockLight(int relX, int y, int relZ)
+	{
+		this.throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(relX, y, relZ);
+		return this.getBlockLightStorage().get(relX, y, relZ);
+	}
+	
+	@Override
+	public int getSkyLight(int relX, int y, int relZ)
+	{
+		this.throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(relX, y, relZ);
+		return this.getSkyLightStorage().get(relX, y, relZ);
+	}
+	
+	@Override
+	public ArrayList<DhBlockPos> getBlockLightPosList() { return this.blockLightPosList; }
+	
+	@Override
+	public boolean doNearbyChunksExist() { return false; }
+	
+	@Override
+	public String toString() { return this.chunkPos.toString(); }
+	
+	@Override
+	public IBlockStateWrapper getBlockState(int relX, int relY, int relZ)
+	{
+		this.throwIndexOutOfBoundsIfRelativePosOutsideChunkBounds(relX, relY, relZ);
+
+		int opacity = this.blockOpacityStorage.get(new DhBlockPos(relX, relY, relZ).hashCode());
+		int lightEmission = this.blockEmissionStorage.get(new DhBlockPos(relX, relY, relZ).hashCode());
+		return new LightingTestBlockStateWrapper(opacity, lightEmission);
+	}
+	
+	@Override
+	public boolean isStillValid() { return true; }
+	
+	
+	
+}

core/src/test/java/tests/LightingEngineTest.java

@@ -0,0 +1,96 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020-2023 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package tests;
+
+import com.seibel.distanthorizons.core.generation.DhLightingEngine;
+import com.seibel.distanthorizons.core.pos.DhChunkPos;
+import com.seibel.distanthorizons.core.util.LodUtil;
+import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
+import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
+import testItems.lightingEngine.LightingTestChunkWrapper;
+
+import java.io.File;
+import java.util.ArrayList;
+
+/**
+ * Can be used to A/B Test lighting engine performance changes. <br><br>
+ *
+ * normal                   - chunks: [1595], total Time: [1490], avg time: [0.9341692789968652] <br>
+ * only surface light prop  - chunks: [1595], total Time: [984], avg time: [0.6169278996865204] <br>
+ * 
+ * @author James Seibel
+ * @version 2024-6-11
+ */
+public class LightingEngineTest
+{
+	/**
+	 * There should be test data in the following core repo folder: <br>
+	 * <code> Core\_Misc Files\test files\Lighting engine test chunk data.7z </code>
+	 */
+	public static final String TEST_DATA_PATH = "C:/Users/James_Seibel/Desktop/test chunk data";
+	
+	
+	//@Test
+	public void TestLightingEngine() throws DataCorruptedException
+	{
+		long totalNanoTime = 0;
+		int chunkCount = 0;
+		
+		File testFolder = new File(TEST_DATA_PATH);
+		File[] chunkFiles = testFolder.listFiles();
+		for (int i = 0; i < chunkFiles.length; i++)
+		{
+			// chunk file parsing //
+			
+			File chunkFile = chunkFiles[i];
+			
+			String fileName = chunkFile.getName(); // C[0,-3]
+			fileName = fileName.replace("C", "").replace("[", "").replace("]", "");
+			int xPos = Integer.parseInt(fileName.split(",")[0]);
+			int zPos = Integer.parseInt(fileName.split(",")[1]);
+			DhChunkPos pos = new DhChunkPos(xPos, zPos);
+			
+			if (i % 100 == 0)
+			{
+				System.out.println(i + "/" + chunkFiles.length);
+			}
+			
+			LightingTestChunkWrapper chunk = new LightingTestChunkWrapper(pos, chunkFile);
+			chunkCount++;
+			
+			ArrayList<IChunkWrapper> nearbyChunkList = new ArrayList<>(1);
+			nearbyChunkList.add(chunk);
+			
+			
+			
+			// lighting //
+			
+			long startTime = System.nanoTime();
+			DhLightingEngine.INSTANCE.lightChunk(chunk, nearbyChunkList, LodUtil.MAX_MC_LIGHT);
+			long endTime = System.nanoTime();
+			totalNanoTime += endTime - startTime;
+		}
+		long timeMs = totalNanoTime / 1_000_000;
+		
+		
+		System.out.println("chunks: ["+chunkCount+"], total Time: ["+timeMs+"], avg time: ["+(timeMs/(double)chunkCount)+"]");
+	}
+	
+}