speed up initial loading when DB migration is necessary

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java

@@ -317,10 +317,42 @@ public class FullDataSourceProviderV2
 		String dimensionName = this.level.getLevelWrapper().getDimensionType().getDimensionName();
 		LOGGER.info("Attempting to migrate data sources for: ["+dimensionName+"]-["+this.saveDir+"]...");
 		
+		
+		
+		//============================//
+		// delete unused data sources //
+		//============================//
+		
+		// this could be done all at once via SQL, 
+		// but doing it in chunks prevents locking the database for long periods of time 
+		long unusedCount = 0;
+		long totalUnusedCount = this.legacyFileHandler.repo.getUnusedDataSourceCount();
+		if (totalUnusedCount != 0)
+		{
+			LOGGER.info("deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources...");
+
+			ArrayList<String> unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100);
+			while (unusedDataPosList.size() != 0)
+			{
+				this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList);
+				unusedCount += unusedDataPosList.size();
+				unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100);
+
+				LOGGER.info("Deleting [" + dimensionName + "] - [" + unusedCount + "/" + totalUnusedCount + "]...");
+			}
+
+			LOGGER.info("Done deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources.");
+		}
+		
+		
+		
+		//=========//
+		// migrate //
+		//=========//
+		
 		int totalCount = this.legacyFileHandler.getDataSourceMigrationCount();
 		LOGGER.info("Found ["+totalCount+"] data sources that need migration.");
 		
-		
 		ArrayList<FullDataSourceV1> legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT);
 		if (!legacyDataSourceList.isEmpty())
 		{

core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java

@@ -22,6 +22,7 @@ package com.seibel.distanthorizons.core.sql.repo;
 import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
+import com.seibel.distanthorizons.coreapi.util.StringUtil;
 
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
@@ -234,4 +235,51 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<DhSectionPos, FullDataS
 	}
 	
 	
+	
+	//======================//
+	// migration - deletion //
+	//======================//
+	
+	/** returns the number of data sources that should be deleted */
+	public long getUnusedDataSourceCount()
+	{
+		Map<String, Object> resultMap = this.queryDictionaryFirst(
+				"select Count(*) as unusedCount from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource'");
+		
+		if (resultMap != null)
+		{
+			// Number cast is necessary because the returned number can be an int or long
+			Number resultNumber = (Number) resultMap.get("unusedCount");
+			long count = resultNumber.longValue();
+			return count;
+		}
+		else
+		{
+			return 0;
+		}
+	}
+	
+	/** Returns single quote surrounded {@link DhSectionPos} serailzed values */
+	public ArrayList<String> getUnusedDataSourcePositionStringList(int deleteCount)
+	{
+		List<Map<String, Object>> deletePosResultMapList = this.queryDictionary(
+				"select DhSectionPos from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource' limit "+deleteCount);
+		
+		ArrayList<String> deletePosList = new ArrayList<>();
+		for (Map<String, Object> deletePosMap : deletePosResultMapList)
+		{
+			String posString = (String) deletePosMap.get("DhSectionPos");
+			deletePosList.add("'"+posString+"'");
+		}
+		
+		return deletePosList;
+	}
+	
+	/** Expects positions to already be surrounded in single quotes */
+	public void deleteUnusedLegacyData(ArrayList<String> deletePosList)
+	{
+		String sectionPosCsv = StringUtil.join(",", deletePosList);
+		this.queryDictionaryFirst("delete from " + this.getTableName() + " where DhSectionPos in (" + sectionPosCsv + ")");
+	}
+	
 }

core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql

@@ -7,12 +7,6 @@ ALTER TABLE Legacy_FullData_V1 ADD COLUMN MigrationFailed BIT NOT NULL DEFAULT 0
 
 --batch--
 
--- we only want to convert the level 0 LOD data, the rest can be generated later
-delete from Legacy_FullData_V1
-where DataType <> 'CompleteFullDataSource' or DataDetailLevel <> 0;
-
---batch--
-
 CREATE TABLE FullData ( 
     -- compound primary key
      DetailLevel TINYINT NOT NULL -- LOD detail level, not section detail level IE 0, 1, 2 not 6, 7, 8