Add Legacy data source migration

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java

@@ -25,6 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColum
 import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder;
 import com.seibel.distanthorizons.core.file.IDataSource;
 import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler;
+import com.seibel.distanthorizons.core.generation.DhLightingEngine;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
@@ -32,6 +33,7 @@ import com.seibel.distanthorizons.core.util.FullDataPointUtil;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
 import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
+import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
 import org.apache.logging.log4j.Logger;
 
@@ -54,6 +56,7 @@ public class NewFullDataSource implements IDataSource<IDhLevel>
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	/** useful for debugging, but can slow down update operations quite a bit due to being called so often. */
 	private static final boolean RUN_UPDATE_DEV_VALIDATION = false; //ModInfo.IS_DEV_BUILD;
+	private static final boolean RUN_V1_MIGRATION_VALIDATION = false;
 	
 	/** measured in data columns */
 	public static final int WIDTH = 64;
@@ -128,25 +131,117 @@ public class NewFullDataSource implements IDataSource<IDhLevel>
 	
 	public static NewFullDataSource createFromCompleteDataSource(CompleteFullDataSource legacyData)
 	{
+		if (CompleteFullDataSource.WIDTH != WIDTH)
+		{
+			throw new UnsupportedOperationException(
+					"Unable to convert CompleteFullDataSource into NewFullDataSource. " +
+					"Data sources have different data point widths and no converter is present. " +
+					"CompleteFullDataSource width ["+CompleteFullDataSource.WIDTH+"], NewFullDataSource width ["+WIDTH+"].");
+		}
+		
+		
+		// Note: this logic only works if the data point data is the same between both versions
 		byte[] columnGenerationSteps = new byte[WIDTH * WIDTH];
 		long[][] dataPoints = new long[WIDTH * WIDTH][];
 		for (int x = 0; x < WIDTH; x++)
 		{
 			for (int z = 0; z < WIDTH; z++)
 			{
-				int index = relativePosToIndex(x, z);
-				
 				SingleColumnFullDataAccessor accessor = legacyData.get(x, z);
-				
 				if (accessor.doesColumnExist())
 				{
+					int index = relativePosToIndex(x, z);
 					dataPoints[index] = accessor.getRaw();
-					columnGenerationSteps[index] = legacyData.getWorldGenStep().value;
+					
+					// reverse the array so index 0 is the lowest,
+					// this is necessary for later logic
+					// source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java
+					long[] dataColumn = dataPoints[index];
+					for(int i = 0; i < dataColumn.length / 2; i++)
+					{
+						long temp = dataColumn[i];
+						dataColumn[i] = dataColumn[dataColumn.length - i - 1];
+						dataColumn[dataColumn.length - i - 1] = temp;
+					}
+					
+					
+					// convert the data point format
+					boolean columnHasNonAirBlock = false;
+					for (int i = 0; i < dataColumn.length; i++)
+					{
+						long dataPoint = dataColumn[i];
+						
+						int id = FullDataPointUtil.getId(dataPoint);
+						int height = FullDataPointUtil.getHeight(dataPoint);
+						int bottomY = FullDataPointUtil.getBottomY(dataPoint);
+						byte blockLight = (byte) FullDataPointUtil.getBlockLight(dataPoint);
+						byte skyLight = (byte) FullDataPointUtil.getSkyLight(dataPoint);
+						
+						long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, skyLight, blockLight);
+						dataColumn[i] = newDataPoint;
+						
+						
+						// check if this datapoint is air
+						if (!columnHasNonAirBlock)
+						{
+							IBlockStateWrapper blockState = legacyData.getMapping().getBlockStateWrapper(id);
+							if (!blockState.isAir())
+							{
+								columnHasNonAirBlock = true;
+							}
+						}
+					}
+					
+					// the old data sources didn't have a generation step written down
+					// if the column has any data points, assume it's fully generated, otherwise assume it's empty
+					columnGenerationSteps[index] = (columnHasNonAirBlock ? EDhApiWorldGenerationStep.LIGHT.value : EDhApiWorldGenerationStep.EMPTY.value);
 				}
 			}
 		}
 		
-		return NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.getMapping(), dataPoints, columnGenerationSteps); 
+		NewFullDataSource newFullDataSource = NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.getMapping(), dataPoints, columnGenerationSteps);
+		
+		
+		// should only be used if debugging, this is a very expensive operation
+		if (RUN_V1_MIGRATION_VALIDATION)
+		{
+			for (int x = 0; x < WIDTH; x++)
+			{
+				for (int z = 0; z < WIDTH; z++)
+				{
+					SingleColumnFullDataAccessor legacyAccessor = legacyData.get(x, z);
+					if (legacyAccessor.doesColumnExist())
+					{
+						SingleColumnFullDataAccessor newAccessor = newFullDataSource.get(x, z);
+						
+						if (newAccessor == null)
+						{
+							LodUtil.assertNotReach("Accessor column mismatch");
+						}
+						else if (legacyAccessor.getRaw().length != newAccessor.getRaw().length)
+						{
+							LodUtil.assertNotReach("Accessor column length mismatch");
+						}
+						else
+						{
+							long[] legacyRaw = legacyAccessor.getRaw();
+							long[] newRaw = newAccessor.getRaw();
+							
+							for (int i = 0; i < legacyRaw.length; i++)
+							{
+								if (legacyRaw[i] != newRaw[i])
+								{
+									LodUtil.assertNotReach("Data mismatch");
+								}
+							}
+						}
+						
+					}
+				}
+			}
+		}
+		
+		return newFullDataSource;
 	}
 	
 	
@@ -286,7 +381,8 @@ public class NewFullDataSource implements IDataSource<IDhLevel>
 			for (int z = 0; z < WIDTH; z += 2)
 			{
 				long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z);
-				byte inputGenStep = inputDataSource.columnGenerationSteps[0]; // TODO
+				// TODO
+				byte inputGenStep = inputDataSource.columnGenerationSteps[0];
 
 
 				int recipientX = (x / 2) + recipientOffsetX;
@@ -349,6 +445,7 @@ public class NewFullDataSource implements IDataSource<IDhLevel>
 			{
 				for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++)
 				{
+					// TODO throw an assertion if the column isn't in order or just fix it...
 					long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)];
 					if (inputDataArray == null || inputDataArray.length == 0)
 					{
@@ -556,7 +653,7 @@ public class NewFullDataSource implements IDataSource<IDhLevel>
 	// helper methods //
 	//================//
 	
-	// TODO make private, any external logic should go through a method, not interact with the arrays directly
+	// TODO  make private, any external logic should go through a method, not interact with the arrays directly
 	public static int relativePosToIndex(int relX, int relZ) throws IndexOutOfBoundsException
 	{ 
 		if (relX < 0 || relZ < 0 ||

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java

@@ -19,25 +19,22 @@
 
 package com.seibel.distanthorizons.core.file.fullDatafile;
 
-import com.seibel.distanthorizons.core.config.Config;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
 import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler;
 import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
-import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
-import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
 import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo;
-import com.seibel.distanthorizons.core.sql.repo.FullDataRepo;
+import com.seibel.distanthorizons.core.sql.repo.LegacyFullDataRepo;
 import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
 
-import java.awt.*;
 import java.io.File;
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.ArrayList;
 
 public class LegacyFullDataFileHandler 
 		extends AbstractLegacyDataSourceHandler<CompleteFullDataSource, IDhLevel> 
@@ -50,7 +47,6 @@ public class LegacyFullDataFileHandler
 	// constructor //
 	//=============//
 	
-	public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); }
 	public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) 
 	{
 		super(level, saveStructure, saveDirOverride);
@@ -67,7 +63,7 @@ public class LegacyFullDataFileHandler
 	{
 		try
 		{
-			return new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
+			return new LegacyFullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
 		}
 		catch (SQLException e)
 		{
@@ -87,18 +83,12 @@ public class LegacyFullDataFileHandler
 	/** Creates a new data source using any DTOs already present in the database. */
 	@Deprecated
 	@Override
-	protected CompleteFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos)
-	{
-		throw new UnsupportedOperationException("Deprecated");
-	}
+	protected CompleteFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) { return null; }
 	
 	
 	@Deprecated
 	@Override
-	protected CompleteFullDataSource makeEmptyDataSource(DhSectionPos pos)
-	{
-		throw new UnsupportedOperationException("Deprecated");
-	}
+	protected CompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { return null; }
 	
 	
 	
@@ -109,8 +99,33 @@ public class LegacyFullDataFileHandler
 	@Deprecated
 	@Override
 	public void writeDataSourceToFile(CompleteFullDataSource fullDataSource) throws IOException
+	{ throw new UnsupportedOperationException("Deprecated"); }
+	
+	
+	
+	//===========//
+	// migration //
+	//===========//
+	
+	public int getDataSourceMigrationCount()
+	{ return ((LegacyFullDataRepo) this.repo).getMigrationCount(); }
+	
+	public ArrayList<CompleteFullDataSource> getDataSourcesToMigrate(int limit)
 	{
-		throw new UnsupportedOperationException("Deprecated");
+		ArrayList<CompleteFullDataSource> dataSourceList = new ArrayList<>();
+		
+		ArrayList<DhSectionPos> migrationPosList = ((LegacyFullDataRepo) this.repo).getPositionsToMigrate(limit);
+		for (int i = 0; i < migrationPosList.size(); i++)
+		{
+			DhSectionPos pos = migrationPosList.get(i);
+			CompleteFullDataSource dataSource = this.get(pos);
+			if (dataSource != null)
+			{
+				dataSourceList.add(dataSource);
+			}
+		}
+		
+		return dataSourceList;
 	}
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java

@@ -20,6 +20,7 @@
 package com.seibel.distanthorizons.core.file.fullDatafile;
 
 import com.seibel.distanthorizons.core.config.Config;
+import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource;
 import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
 import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler;
@@ -29,7 +30,6 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
 import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
 import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO;
-import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
 import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo;
 import com.seibel.distanthorizons.core.util.ThreadUtil;
 import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
@@ -42,7 +42,9 @@ import java.io.IOException;
 import java.sql.SQLException;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantLock;
 
 public class NewFullDataFileHandler 
@@ -51,6 +53,10 @@ public class NewFullDataFileHandler
 {
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	
+	/** how many data sources should be pulled down for migration at once */
+	private static final int MIGRATION_BATCH_COUNT = 20;
+	private static final String MIGRATION_THREAD_NAME_PREFIX = "Full Data Migration Thread: ";
+	
 	protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50;
 	/** how many parent update tasks can be in the queue at once */
 	protected static final int MAX_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get();
@@ -59,6 +65,18 @@ public class NewFullDataFileHandler
 	protected static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250;
 	
 	
+	protected final ThreadPoolExecutor migrationThreadPool;
+	/** 
+	 * Interrupting the migration thread pool doesn't work well and may corrupt the database
+	 * vs gracefully shutting down the thread ourselves. 
+	 */
+	protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true);
+	protected final LegacyFullDataFileHandler legacyFileHandler;
+	
+	/** 
+	 * Tracks which positions are currently being updated
+	 * to prevent duplicate concurrent updates.
+	 */
 	public final Set<DhSectionPos> parentUpdatingPosSet = ConcurrentHashMap.newKeySet();
 	
 	// TODO only run thread if modifications happened recently
@@ -78,10 +96,18 @@ public class NewFullDataFileHandler
 	public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) 
 	{
 		super(level, saveStructure, saveDirOverride);
+		this.legacyFileHandler = new LegacyFullDataFileHandler(level, saveStructure, saveDirOverride);
 		
 		DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus);
 		
 		String dimensionName = level.getLevelWrapper().getDimensionType().getDimensionName();
+		
+		// start migrating any legacy data sources present in the background
+		int totalCount = this.legacyFileHandler.getDataSourceMigrationCount();
+		LOGGER.info("Found ["+totalCount+"] data sources that need migration.");
+		this.migrationThreadPool = ThreadUtil.makeRateLimitedThreadPool(1, MIGRATION_THREAD_NAME_PREFIX +"["+dimensionName+"]", Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads.get(), Thread.MIN_PRIORITY, (Semaphore)null);
+		this.migrationThreadPool.execute(() -> this.convertLegacyDataSources());
+		
 		this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Parent Update Queue ["+dimensionName+"]");
 		this.updateQueueProcessor.execute(() -> this.runUpdateQueue());
 	}
@@ -156,13 +182,14 @@ public class NewFullDataFileHandler
 				
 				
 				
-				// only add more items to the queue if half or more of the previous tasks have been completed
+				// queue parent updates
 				if (executor.getQueue().size() < MAX_UPDATE_TASK_COUNT
 					&& this.parentUpdatingPosSet.size() < MAX_UPDATE_TASK_COUNT)
 				{
 					// get the positions that need to be applied to their parents
 					ArrayList<DhSectionPos> parentUpdatePosList = this.repo.getPositionsToUpdate(MAX_UPDATE_TASK_COUNT);
 					
+					// combine updates together based on their parent
 					HashMap<DhSectionPos, HashSet<DhSectionPos>> updatePosByParentPos = new HashMap<>();
 					for (DhSectionPos pos : parentUpdatePosList)
 					{
@@ -177,9 +204,7 @@ public class NewFullDataFileHandler
 						});
 					}
 					
-					
-					
-					// queue each update
+					// queue the updates
 					for (DhSectionPos parentUpdatePos : updatePosByParentPos.keySet())
 					{
 						// stop if there are already a bunch of updates queued
@@ -250,6 +275,7 @@ public class NewFullDataFileHandler
 						}
 					}
 				}
+				
 			}
 			catch (InterruptedException ignored) { Thread.currentThread().interrupt(); }
 			catch (Exception e)
@@ -263,6 +289,63 @@ public class NewFullDataFileHandler
 	
 	
 	
+	//=======================//
+	// data source migration //
+	//=======================//
+	
+	private void convertLegacyDataSources()
+	{
+		String dimensionName = this.level.getLevelWrapper().getDimensionType().getDimensionName();
+		LOGGER.info("Attempting to migrate data sources for: ["+dimensionName+"]-["+this.saveDir+"]...");
+		
+		int totalCount = this.legacyFileHandler.getDataSourceMigrationCount();
+		LOGGER.info("Found ["+totalCount+"] data sources that need migration.");
+		
+		
+		ArrayList<CompleteFullDataSource> legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT);
+		if (!legacyDataSourceList.isEmpty())
+		{
+			// keep going until every data source has been migrated
+			int progressCount = 0;
+			while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get())
+			{
+				LOGGER.info("Migrating ["+dimensionName+"] - [" + progressCount + "/" + totalCount + "]...");
+				
+				for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++)
+				{
+					// convert the legacy data source to the new format
+					CompleteFullDataSource legacyDataSource = legacyDataSourceList.get(i);
+					NewFullDataSource newDataSource = NewFullDataSource.createFromCompleteDataSource(legacyDataSource);
+					newDataSource.applyToParent = true;
+					
+					this.updateDataSourceAtPos(newDataSource.getSectionPos(), newDataSource, true);
+					
+					// the legacy data source can now be deleted
+					this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getSectionPos());
+				}
+				
+				legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT);
+				progressCount += legacyDataSourceList.size();
+			}
+			
+			
+			if (this.migrationThreadRunning.get())
+			{
+				LOGGER.info("migration complete for: ["+dimensionName+"]-["+this.saveDir+"].");
+			}
+			else
+			{
+				LOGGER.info("migration stopped for: ["+dimensionName+"]-["+this.saveDir+"].");
+			}
+		}
+		else
+		{
+			LOGGER.info("No migration necessary.");
+		}
+	}
+	
+	
+	
 	//===========//
 	// overrides //
 	//===========//
@@ -284,6 +367,9 @@ public class NewFullDataFileHandler
 	{
 		super.close();
 		this.updateQueueProcessor.shutdownNow();
+		
+		this.migrationThreadRunning.set(false);
+		this.migrationThreadPool.shutdown();
 	}
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataRepo.java

@@ -1,43 +0,0 @@
-/*
- *    This file is part of the Distant Horizons mod
- *    licensed under the GNU LGPL v3 License.
- *
- *    Copyright (C) 2020-2023 James Seibel
- *
- *    This program is free software: you can redistribute it and/or modify
- *    it under the terms of the GNU Lesser General Public License as published by
- *    the Free Software Foundation, version 3.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *    GNU Lesser General Public License for more details.
- *
- *    You should have received a copy of the GNU Lesser General Public License
- *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.seibel.distanthorizons.core.sql.repo;
-
-import com.seibel.distanthorizons.core.pos.DhSectionPos;
-
-import java.sql.SQLException;
-
-public class FullDataRepo extends AbstractLegacyDataSourceRepo
-{
-	public static final String TABLE_NAME = "DhFullData";
-	
-	
-	public FullDataRepo(String databaseType, String databaseLocation) throws SQLException
-	{
-		super(databaseType, databaseLocation);
-	}
-	
-	
-	@Override
-	public String getTableName() { return TABLE_NAME; }
-	
-	@Override 
-	public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; }
-	
-}

core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java

@@ -0,0 +1,91 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020-2023 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.seibel.distanthorizons.core.sql.repo;
+
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo
+{
+	public static final String TABLE_NAME = "Legacy_FullData_V1";
+	
+	
+	public LegacyFullDataRepo(String databaseType, String databaseLocation) throws SQLException
+	{
+		super(databaseType, databaseLocation);
+	}
+	
+	
+	@Override
+	public String getTableName() { return TABLE_NAME; }
+	
+	@Override 
+	public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; }
+	
+	
+	
+	//===========//
+	// migration //
+	//===========//
+	
+	/** Returns how many positions need to be migrated over to the new version */
+	public int getMigrationCount()
+	{
+		Map<String, Object> resultMap = this.queryDictionaryFirst(
+				"select COUNT(*) as itemCount from "+this.getTableName());
+		
+		if (resultMap == null)
+		{
+			return 0;
+		}
+		else
+		{
+			int count = (int) resultMap.get("itemCount");
+			return count;
+		}
+	}
+	
+	/** Returns the new "returnCount" positions that need to be migrated */
+	public ArrayList<DhSectionPos> getPositionsToMigrate(int returnCount)
+	{
+		ArrayList<DhSectionPos> list = new ArrayList<>();
+		
+		List<Map<String, Object>> resultMapList = this.queryDictionary(
+				"select DhSectionPos " +
+						"from "+this.getTableName()+" " +
+						"LIMIT "+returnCount+";");
+		
+		for (Map<String, Object> resultMap : resultMapList)
+		{
+			// returned in the format [sectionDetailLevel,x,z] IE [6,0,0]
+			DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos"));
+			list.add(sectionPos);
+		}
+		
+		return list;
+	}
+	
+	
+	
+}

core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java

@@ -80,9 +80,9 @@ public class ThreadUtil
 	
 	
 	/** should only be used if there isn't a config controlling the run time ratio of this thread pool */
-	public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, Double runTimeRatio, int relativePriority, Semaphore activeThreadCountSemaphore) 
+	public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, Double runTimeRatio, int threadPriority, Semaphore activeThreadCountSemaphore) 
 	{
-		return new RateLimitedThreadPoolExecutor(poolSize, runTimeRatio, new DhThreadFactory(name, Thread.NORM_PRIORITY + relativePriority), activeThreadCountSemaphore);
+		return new RateLimitedThreadPoolExecutor(poolSize, runTimeRatio, new DhThreadFactory(name, threadPriority), activeThreadCountSemaphore);
 	}
 	public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, Double runTimeRatio, DhThreadFactory threadFactory, Semaphore activeThreadCountSemaphore) 
 	{

core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql

@@ -1,5 +1,16 @@
 
-select * from DhRenderData; -- here to prevent crashing when running the first batch
+ALTER TABLE `DhFullData` RENAME TO `Legacy_FullData_V1`;
+
+--batch--
+
+-- we only want to convert the level 0 LOD data, the rest can be generated later
+delete from Legacy_FullData_V1
+where DataType <> 'CompleteFullDataSource' or DataDetailLevel <> 0;
+
+--batch--
+
+-- shrink the database file for the removed legacy detail levels
+VACUUM;
 
 --batch--