Remove unused ID mappings after data update

Requires re-downsampling all LODs

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java

@@ -78,7 +78,9 @@ public class FullDataSourceV2
 	/** how many chunks wide this datasource is at detail level 0. */
 	public static final int NUMB_OF_CHUNKS_WIDE = WIDTH / LodUtil.CHUNK_WIDTH;
 	
-	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("FullDataV2");
+	private static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("FullDataV2");
+	
+	private static final ThreadLocal<FullDataPointIdMap> DATA_MAP_FOR_REMAPPING_REF = ThreadLocal.withInitial(() -> new FullDataPointIdMap(DhSectionPos.encode((byte)0, 0, 0)));
 	
 	
 	
@@ -441,6 +443,8 @@ public class FullDataSourceV2
 		}
 		
 		
+		// needed to prevent infinite mapped ID growth
+		this.removeUnusedIdsAndRemap();
 		
 		
 		
@@ -1130,6 +1134,68 @@ public class FullDataSourceV2
 		return dataChanged;
 	}
 	
+	/**
+	 * Should be run at the end of {@link FullDataSourceV2#updateFromDataSource}
+	 * so the {@link FullDataPointIdMap} will only contain ID's that are actively in use. <br><br>
+	 * 
+	 * This prevents the {@link FullDataPointIdMap} from growing infinitely when merged.
+	 * 
+	 * @see FullDataPointIdMap#mergeAndReturnRemappedEntityIds(FullDataPointIdMap) 
+	 */
+	private void removeUnusedIdsAndRemap()
+	{
+		Int2IntOpenHashMap newIdByOldId = new Int2IntOpenHashMap();
+		FullDataPointIdMap newMap = DATA_MAP_FOR_REMAPPING_REF.get();
+		newMap.clear(this.pos);
+		
+		
+		// find all the IDs that are currently in use
+		for (int x = 0; x < WIDTH; x++)
+		{
+			for (int z = 0; z < WIDTH; z++)
+			{
+				int index = relativePosToIndex(x, z);
+				LongArrayList dataColumn = this.dataPoints[index];
+				for (int i = 0; i < dataColumn.size(); i++)
+				{
+					long dataPoint = dataColumn.getLong(i);
+					int oldId = FullDataPointUtil.getId(dataPoint);
+					
+					int newId = newMap.addIfNotPresentAndGetId(
+						this.mapping.getBiomeWrapper(oldId),
+						this.mapping.getBlockStateWrapper(oldId));
+					
+					newIdByOldId.put(oldId, newId);
+				}
+			}
+		}
+		
+		
+		// replace the old entries to remove any unneeded ones
+		this.mapping.clear(this.pos);
+		this.mapping.addAll(newMap);
+		
+		
+		// remap the data
+		for (int x = 0; x < WIDTH; x++)
+		{
+			for (int z = 0; z < WIDTH; z++)
+			{
+				int index = relativePosToIndex(x, z);
+				LongArrayList dataColumn = this.dataPoints[index];
+				for (int i = 0; i < dataColumn.size(); i++)
+				{
+					long oldDataPoint = dataColumn.getLong(i);
+					int oldId = FullDataPointUtil.getId(oldDataPoint);
+					
+					int newId = newIdByOldId.get(oldId);
+					long newDataPoint = FullDataPointUtil.setId(oldDataPoint, newId);
+					dataColumn.set(i, newDataPoint);
+				}
+			}
+		}
+	}
+	
 	//endregion
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java

@@ -477,7 +477,10 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
 			"   abs((PosX << (6 + DetailLevel)) - ?) + abs((PosZ << (6 + DetailLevel)) - ?) AS Distance " +
 			"FROM " + this.getTableName() + " " +
 			"WHERE ApplyToParent = 1 " +
-			"ORDER BY DetailLevel ASC, Distance ASC " +
+			// sorting by detail level first will reduce the total number of updates required
+			// but makes it feel less responsive since distant updates won't be seen until everything 
+			// underneith is done
+			"ORDER BY Distance ASC " + // DetailLevel ASC,
 			"LIMIT ?; ";
 	public LongArrayList getPositionsToUpdate(int targetBlockPosX, int targetBlockPosZ, int returnCount)
 	{ return this.getPositionsToUpdate(targetBlockPosX, targetBlockPosZ, returnCount, true); }
@@ -488,7 +491,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
 			"   abs((PosX << (6 + DetailLevel)) - ?) + abs((PosZ << (6 + DetailLevel)) - ?) AS Distance " +
 			"FROM " + this.getTableName() + " " +
 			"WHERE ApplyToChildren = 1 " +
-			"ORDER BY DetailLevel ASC, Distance ASC " +
+			"ORDER BY Distance ASC " + // DetailLevel ASC, 
 			"LIMIT ?; ";
 	public LongArrayList getChildPositionsToUpdate(int targetBlockPosX, int targetBlockPosZ, int returnCount)
 	{ return this.getPositionsToUpdate(targetBlockPosX, targetBlockPosZ, returnCount, false); }

core/src/main/resources/sqlScripts/0090-sqlite-addAdjacentFullDataColumns.sql

@@ -9,5 +9,4 @@ alter table FullData add column SouthAdjData BLOB NULL;
 --batch--
 alter table FullData add column EastAdjData BLOB NULL;
 --batch--
-alter table FullData add column WestAdjData BLOB NULL;
---batch--
+alter table FullData add column WestAdjData BLOB NULL;

core/src/main/resources/sqlScripts/0100-sqlite-deleteLowDetailDataForRegen.sql

@@ -0,0 +1,13 @@
+
+-- This is done to fix a bug where a lot of unnecessary
+-- ID mapping data is saved, which significantly reduces
+-- loading/deserializing/decompression time
+
+
+-- delete all data above 0 (max detail)
+-- so it can be re-created
+delete from FullData where DetailLevel > 0;
+--batch--
+
+-- re-downsample all LOD data
+update FullData set ApplyToParent = 1;

core/src/main/resources/sqlScripts/scriptList.txt

@@ -9,3 +9,4 @@
 0070-sqlite-createBeaconBeamTable.sql
 0080-sqlite-addApplyToChildrenColumn.sql
 0090-sqlite-addAdjacentFullDataColumns.sql
+0100-sqlite-deleteLowDetailDataForRegen.sql