diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiDebugRendering.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiDebugRendering.java
index 1500aa13f..b867c51a7 100644
--- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiDebugRendering.java
+++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiDebugRendering.java
@@ -43,11 +43,7 @@ public enum EDhApiDebugRendering
SHOW_BLOCK_MATERIAL,
/** Only draw overlapping LOD quads. */
- SHOW_OVERLAPPING_QUADS,
-
- /** LOD colors are based on renderSource flags. */
- SHOW_RENDER_SOURCE_FLAG;
-
+ SHOW_OVERLAPPING_QUADS;
public static EDhApiDebugRendering next(EDhApiDebugRendering type)
{
@@ -60,7 +56,7 @@ public enum EDhApiDebugRendering
case SHOW_BLOCK_MATERIAL:
return SHOW_OVERLAPPING_QUADS;
case SHOW_OVERLAPPING_QUADS:
- return SHOW_RENDER_SOURCE_FLAG;
+ return OFF;
default:
return OFF;
}
@@ -71,8 +67,6 @@ public enum EDhApiDebugRendering
switch (type)
{
case OFF:
- return SHOW_RENDER_SOURCE_FLAG;
- case SHOW_RENDER_SOURCE_FLAG:
return SHOW_OVERLAPPING_QUADS;
case SHOW_OVERLAPPING_QUADS:
return SHOW_DETAIL;
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java
index 3ec642291..834182cf6 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java
@@ -2,11 +2,14 @@ package com.seibel.distanthorizons.core.dataObjects;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
+import com.seibel.distanthorizons.core.pooling.StringPool;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
+import it.unimi.dsi.fastutil.chars.CharArrayList;
import java.util.concurrent.ConcurrentHashMap;
@@ -121,9 +124,9 @@ public class BlockBiomeWrapperPair
- //=================//
- // (de)serializing //
- //=================//
+ //=============//
+ // serializing //
+ //=============//
public String serialize()
{
@@ -135,17 +138,6 @@ public class BlockBiomeWrapperPair
return this.serialString;
}
- public static BlockBiomeWrapperPair deserialize(String str, ILevelWrapper levelWrapper) throws DataCorruptedException
- {
- int separatorIndex = str.indexOf(FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING);
- if (separatorIndex == -1)
- {
- throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry ["+str+"], unable to find separator.");
- }
-
- IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapperOrGetDefault(str.substring(0, separatorIndex), levelWrapper);
- IBlockStateWrapper blockState = WRAPPER_FACTORY.deserializeBlockStateWrapperOrGetDefault(str.substring(separatorIndex+FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING.length()), levelWrapper);
- return BlockBiomeWrapperPair.get(blockState, biome);
- }
+
}
\ No newline at end of file
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java
index 6ab7f07c2..081f814fb 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java
@@ -20,37 +20,42 @@
package com.seibel.distanthorizons.core.dataObjects.fullData;
import com.seibel.distanthorizons.core.dataObjects.BlockBiomeWrapperPair;
+import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
+import com.seibel.distanthorizons.core.pooling.StringPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
+import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.core.logging.DhLogger;
+import it.unimi.dsi.fastutil.chars.CharArrayList;
+import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
- * WARNING: This is not THREAD-SAFE!
- *
- * Used to map a numerical IDs to a Biome/BlockState pair.
- *
- * TODO the serializing of this map might be really big
- * since it stringifies every block and biome name, which is quite bulky.
- * It might be worth while to have a biome and block ID that then both get mapped
- * to the data point ID to reduce file size.
- * And/or it would be good to dynamically remove IDs that aren't currently in use.
+ * Used to map a numerical IDs to a Biome/BlockState pair.
+ * Note: This is not thread safe.
*
* @author Leetom
*/
public class FullDataPointIdMap
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
+
+ private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class);
+
+ public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("IdMap");
+
/**
* Should only be enabled when debugging.
* Has the system check if any duplicate Entries were read/written
@@ -249,8 +254,9 @@ public class FullDataPointIdMap
}
}
- /** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */
- public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, long pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
+ /** Clears and populates the given {@link FullDataPointIdMap} from the given UTF formatted stream */
+ public static void deserialize(@NotNull FullDataPointIdMap map, DhDataInputStream inputStream, long pos, ILevelWrapper levelWrapper)
+ throws IOException, InterruptedException, DataCorruptedException
{
int entityCount = inputStream.readInt();
if (entityCount < 0)
@@ -258,45 +264,127 @@ public class FullDataPointIdMap
throw new DataCorruptedException("FullDataPointIdMap deserialize entry count should have a number greater than or equal to 0, returned value ["+entityCount+"].");
}
+ // clearing the old values is necessary so we can re-use the same map multiple times
+ map.clear(pos);
// only used when debugging
HashMap dataPointEntryBySerialization = new HashMap<>();
- FullDataPointIdMap newMap = new FullDataPointIdMap(pos);
- for (int i = 0; i < entityCount; i++)
+ try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutArrays(0, 0, 0, 3))
{
- // necessary to prevent issues with deserializing objects after the level has been closed
- if (Thread.interrupted())
+ for (int i = 0; i < entityCount; i++)
{
- throw new InterruptedException("[" + FullDataPointIdMap.class.getSimpleName() + "] deserializing interrupted.");
- }
-
-
- String entryString = inputStream.readUTF();
- BlockBiomeWrapperPair newPair = BlockBiomeWrapperPair.deserialize(entryString, levelWrapper);
- newMap.blockBiomePairList.add(newPair);
-
- if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
- {
- if (dataPointEntryBySerialization.containsKey(entryString))
+ // necessary to prevent issues with deserializing objects after the level has been closed
+ if (Thread.interrupted())
{
- LOGGER.error("Duplicate deserialized entry found with serial: " + entryString);
+ throw new InterruptedException("[" + FullDataPointIdMap.class.getSimpleName() + "] deserializing interrupted.");
}
- if (dataPointEntryBySerialization.containsValue(newPair))
+
+ int length = inputStream.readUnsignedShort();
+
+ CharArrayList fullCharList = checkout.getCharArray(0, length);
+ CharArrayList biomeCharList = checkout.getCharArray(1, length);
+ CharArrayList blockCharList = checkout.getCharArray(2, length);
+
+ // parse the full UTF string
+ for (int stringIndex = 0; stringIndex < length; stringIndex++)
{
- LOGGER.error("Duplicate deserialized entry found with value: " + newPair.serialize());
+ byte b = inputStream.readByte();
+ char c = (char) (b & 0xFF);
+ fullCharList.set(stringIndex, c);
+ }
+ splitCharArray(fullCharList, biomeCharList, blockCharList);
+
+ String biomeString = StringPool.INSTANCE.getPooledString(biomeCharList);
+ IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapperOrGetDefault(biomeString, levelWrapper);
+
+ String blockStateString = StringPool.INSTANCE.getPooledString(blockCharList);
+ IBlockStateWrapper blockState = WRAPPER_FACTORY.deserializeBlockStateWrapperOrGetDefault(blockStateString, levelWrapper);
+
+ BlockBiomeWrapperPair newPair = BlockBiomeWrapperPair.get(blockState, biome);
+ map.blockBiomePairList.add(newPair);
+
+
+ if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
+ {
+ String entryString = StringPool.INSTANCE.getPooledString(fullCharList);
+ if (dataPointEntryBySerialization.containsKey(entryString))
+ {
+ LOGGER.error("Duplicate deserialized entry found with serial: " + entryString);
+ }
+ if (dataPointEntryBySerialization.containsValue(newPair))
+ {
+ LOGGER.error("Duplicate deserialized entry found with value: " + newPair.serialize());
+ }
+ dataPointEntryBySerialization.put(entryString, newPair);
}
- dataPointEntryBySerialization.put(entryString, newPair);
}
}
- if (newMap.size() != entityCount)
+ if (map.size() != entityCount)
{
// if the mappings are out of sync then the LODs will render incorrectly due to IDs being wrong
- LodUtil.assertNotReach("ID maps failed to deserialize for pos: ["+ DhSectionPos.toString(pos)+"], incorrect entity count. Expected count ["+entityCount+"], actual count ["+newMap.size()+"]");
+ LodUtil.assertNotReach("ID maps failed to deserialize for pos: ["+ DhSectionPos.toString(pos)+"], incorrect entity count. Expected count ["+entityCount+"], actual count ["+map.size()+"]");
+ }
+ }
+ /**
+ * Splits up the given input {@link CharArrayList} into the
+ * necessary biome and blockstate lists based on the location of
+ * {@link FullDataPointIdMap#BLOCK_STATE_SEPARATOR_STRING}
+ */
+ private static void splitCharArray(
+ CharArrayList input,
+ CharArrayList biomeString, CharArrayList blockString) throws DataCorruptedException
+ {
+ boolean separatorFound = false;
+
+ int foundStartIndex = -1;
+ int separatorIndex = 0;
+ for (int inputIndex = 0; inputIndex < input.size(); inputIndex++)
+ {
+ char ch = input.getChar(inputIndex);
+ if (ch == FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING.charAt(separatorIndex))
+ {
+ if (!separatorFound)
+ {
+ foundStartIndex = inputIndex;
+ }
+ separatorFound = true;
+ separatorIndex++;
+ }
+ else
+ {
+ separatorFound = false;
+ foundStartIndex = -1;
+ separatorIndex = 0;
+ }
+
+ if (separatorFound
+ && separatorIndex == FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING.length())
+ {
+ break;
+ }
}
- return newMap;
+
+ if (foundStartIndex == -1)
+ {
+ throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry ["+input.toString()+"], unable to find separator.");
+ }
+
+ biomeString.clear();
+ for (int i = 0; i < foundStartIndex; i++)
+ {
+ biomeString.push(input.getChar(i));
+ }
+
+ blockString.clear();
+ for (int i = foundStartIndex + FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING.length();
+ i < input.size();
+ i++)
+ {
+ blockString.push(input.getChar(i));
+ }
}
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java
index 48d221ba2..36bde4d8e 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java
@@ -355,7 +355,9 @@ public class FullDataSourceV1
throw new IOException("Invalid data content end guard for ID mapping");
}
- return FullDataPointIdMap.deserialize(inputStream, this.pos, levelWrapper);
+ FullDataPointIdMap newMap = new FullDataPointIdMap(this.pos);
+ FullDataPointIdMap.deserialize(newMap, inputStream, this.pos, levelWrapper);
+ return newMap;
}
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
index 836a65834..21a9f6c98 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
@@ -222,7 +222,7 @@ public class FullDataSourceV2
byte @Nullable [] columnGenerationSteps, byte @Nullable [] columnWorldCompressionMode,
boolean empty)
{
- super(ARRAY_LIST_POOL, 2, 0, WIDTH * WIDTH);
+ super(ARRAY_LIST_POOL, 2, 0, WIDTH * WIDTH, 0);
LodUtil.assertTrue(data == null || data.length == WIDTH * WIDTH);
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
index ff7badd15..210820040 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
@@ -53,8 +53,6 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
public final LongArrayList renderDataContainer;
- public final DebugSourceFlag[] debugSourceFlags;
-
private boolean isEmpty = true;
@@ -73,7 +71,7 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
*/
private ColumnRenderSource(long pos, int maxVerticalSize, int yOffset)
{
- super(ARRAY_LIST_POOL, 0, 0, 1);
+ super(ARRAY_LIST_POOL, 0, 0, 1, 0);
this.pos = pos;
this.yOffset = yOffset;
@@ -81,8 +79,6 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
this.verticalDataCount = maxVerticalSize;
this.renderDataContainer = this.pooledArraysCheckout.getLongArray(0, WIDTH * WIDTH * this.verticalDataCount);
-
- this.debugSourceFlags = new DebugSourceFlag[WIDTH * WIDTH];
}
@@ -159,26 +155,6 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
- //=======//
- // debug //
- //=======//
-
- /** Sets the debug flag for the given area */
- public void fillDebugFlag(int xStart, int zStart, int xWidth, int zWidth, DebugSourceFlag flag)
- {
- for (int x = xStart; x < xStart + xWidth; x++)
- {
- for (int z = zStart; z < zStart + zWidth; z++)
- {
- this.debugSourceFlags[x * WIDTH + z] = flag;
- }
- }
- }
-
- public DebugSourceFlag debugGetFlag(int ox, int oz) { return this.debugSourceFlags[ox * WIDTH + oz]; }
-
-
-
//==============//
// base methods //
//==============//
@@ -219,19 +195,4 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
- //==============//
- // helper enums //
- //==============//
-
- public enum DebugSourceFlag
- {
- FULL(ColorUtil.BLUE),
- DIRECT(ColorUtil.WHITE),
- FILE(ColorUtil.BROWN);
-
- public final int color;
-
- DebugSourceFlag(int color) { this.color = color; }
- }
-
}
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java
index a797f18be..f999407e9 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java
@@ -108,7 +108,7 @@ public class ColumnRenderBufferBuilder
//===================//
// pooled arrays for ColumnBox use
- try (PhantomArrayListCheckout phantomArrayCheckout = ARRAY_LIST_POOL.checkoutArrays(0, 0, 2))
+ try (PhantomArrayListCheckout phantomArrayCheckout = ARRAY_LIST_POOL.checkoutLongArrays(2))
{
byte thisDetailLevel = renderSource.getDataDetailLevel();
for (int relX = 0; relX < ColumnRenderSource.WIDTH; relX++)
@@ -117,9 +117,10 @@ public class ColumnRenderBufferBuilder
{
// ignore empty/null columns
ColumnArrayView columnRenderData = renderSource.getVerticalDataPointView(relX, relZ);
- if (columnRenderData.size() == 0
- || !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0))
- || RenderDataPointUtil.hasZeroHeight(columnRenderData.get(0)))
+ if (columnRenderData == null
+ || columnRenderData.size() == 0
+ || !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0))
+ || RenderDataPointUtil.hasZeroHeight(columnRenderData.get(0)))
{
continue;
}
@@ -244,8 +245,6 @@ public class ColumnRenderBufferBuilder
// build this render column //
//==========================//
- ColumnRenderSource.DebugSourceFlag debugSourceFlag = renderSource.debugGetFlag(relX, relZ);
-
for (int i = 0; i < columnRenderData.size(); i++)
{
// can be uncommented to limit which vertical LOD is generated
@@ -276,7 +275,7 @@ public class ColumnRenderBufferBuilder
data, topDataPoint, bottomDataPoint,
adjColumnViews, isSameDetailLevel,
thisDetailLevel, relX, relZ,
- quadBuilder, debugSourceFlag);
+ quadBuilder);
}
}// for z
@@ -290,7 +289,7 @@ public class ColumnRenderBufferBuilder
long renderData, long topRenderData, long bottomRenderData,
ColumnArrayView[] adjColumnViews, boolean[] isSameDetailLevel,
byte detailLevel, int renderSourceOffsetPosX, int renderSourceOffsetPosZ,
- LodQuadBuilder quadBuilder, ColumnRenderSource.DebugSourceFlag debugSource)
+ LodQuadBuilder quadBuilder)
{
long sectionPos = DhSectionPos.encode(detailLevel, renderSourceOffsetPosX, renderSourceOffsetPosZ);
@@ -407,12 +406,6 @@ public class ColumnRenderBufferBuilder
fullBright = true;
break;
}
- case SHOW_RENDER_SOURCE_FLAG:
- {
- color = debugSource == null ? ColorUtil.RED : debugSource.color;
- fullBright = true;
- break;
- }
default:
throw new IllegalArgumentException("Unknown debug mode: " + debugging);
}
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java
index bb4d3d0f0..64ef16159 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java
@@ -137,8 +137,6 @@ public class FullDataToRenderDataTransformer
}
}
- columnSource.fillDebugFlag(0, 0, ColumnRenderSource.WIDTH, ColumnRenderSource.WIDTH, ColumnRenderSource.DebugSourceFlag.FULL);
-
return columnSource;
}
@@ -164,21 +162,16 @@ public class FullDataToRenderDataTransformer
}
else
{
- PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutArrays(0, 0, 1);
- LongArrayList dataArrayList = checkout.getLongArray(0, fullDataLength);
-
- try
+ try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutLongArrays(1))
{
+ LongArrayList dataArrayList = checkout.getLongArray(0, fullDataLength);
+
// expand the ColumnArrayView to fit the new larger max vertical size
ColumnArrayView newColumnArrayView = new ColumnArrayView(dataArrayList, fullDataLength, 0, fullDataLength);
setRenderColumnView(levelWrapper, fullDataSource, blockX, blockZ, newColumnArrayView, fullDataColumn);
columnArrayView.changeVerticalSizeFrom(newColumnArrayView);
}
- finally
- {
- ARRAY_LIST_POOL.returnCheckout(checkout);
- }
}
}
private static void setRenderColumnView(
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java
index 0167dc8ed..38368fe3f 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java
@@ -351,7 +351,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
// don't check any child positions if this position is already fully generated
if (this.repo.existsWithKey(pos))
{
- try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutArrays(1, 0, 0))
+ try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1))
{
ByteArrayList columnGenStepArray = checkout.getByteArray(0, FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH);
this.repo.getColumnGenerationStepForPos(pos, columnGenStepArray);
@@ -398,7 +398,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
{
EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT;
- try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutArrays(1, 0, 0))
+ try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1))
{
ByteArrayList columnGenerationSteps = checkout.getByteArray(0, FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH);
this.repo.getColumnGenerationStepForPos(genPos, columnGenerationSteps);
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java
index c15542e86..0b59bdba2 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java
@@ -1,9 +1,12 @@
package com.seibel.distanthorizons.core.file.fullDatafile.V1;
+import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
@@ -29,6 +32,8 @@ public class FullDataSourceProviderV1
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
+ public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("V1DTO");
+
protected final ReentrantLock closeLock = new ReentrantLock();
protected volatile boolean isShutdown = false;
@@ -64,7 +69,9 @@ public class FullDataSourceProviderV1
protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException, DataCorruptedException
{
FullDataSourceV1 dataSource = FullDataSourceV1.createEmpty(dto.pos);
- try (DhDataInputStream inputStream = dto.getInputStream())
+ try (PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1);
+ // LZ4 was used by DH before 2.1.0 and as such must be used until the data format is changed to record the compressor)
+ DhDataInputStream inputStream = DhDataInputStream.create(dto.dataArray, EDhApiDataCompressionMode.LZ4, checkout))
{
dataSource.populateFromStream(dto, inputStream, this.level);
}
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java
index 31e724299..9a1bee660 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java
@@ -38,18 +38,19 @@ public abstract class AbstractPhantomArrayList implements AutoCloseable
//=============//
/** The Array counts can be 0 or greater. */
- public AbstractPhantomArrayList(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount)
+ public AbstractPhantomArrayList(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount, int charArrayCount)
{
if (byteArrayCount < 0
|| shortArrayCount < 0
- || longArrayCount < 0)
+ || longArrayCount < 0
+ || charArrayCount < 0)
{
throw new IllegalArgumentException("Can't get a negative number of pooled arrays.");
}
this.phantomArrayListPool = phantomArrayListPool;
this.phantomReference = new PhantomReference<>(this, this.phantomArrayListPool.phantomRefQueue);
- this.pooledArraysCheckout = this.phantomArrayListPool.checkoutArrays(byteArrayCount, shortArrayCount, longArrayCount);
+ this.pooledArraysCheckout = this.phantomArrayListPool.checkoutArrays(byteArrayCount, shortArrayCount, longArrayCount, charArrayCount);
this.phantomArrayListPool.phantomRefToCheckout.put(this.phantomReference, this.pooledArraysCheckout);
}
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java
index 2213dd3ef..42545fc01 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java
@@ -3,6 +3,7 @@ package com.seibel.distanthorizons.core.pooling;
import com.seibel.distanthorizons.core.util.ListUtil;
import com.seibel.distanthorizons.coreapi.util.StringUtil;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
+import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.jetbrains.annotations.NotNull;
@@ -13,6 +14,9 @@ import java.util.ArrayList;
import java.util.Arrays;
/**
+ * TODO move into util.objects.pooling
+ * TODO split up trackable objects into inner objects to remove duplicate code
+ *
* This keeps track of all the poolable
* arrays that can be retrieved via the {@link PhantomArrayListPool}.
*
@@ -39,6 +43,7 @@ public class PhantomArrayListCheckout implements AutoCloseable
private final ArrayList byteArrayLists = new ArrayList<>();
private final ArrayList shortArrayLists = new ArrayList<>();
private final ArrayList longArrayLists = new ArrayList<>();
+ private final ArrayList charArrayLists = new ArrayList<>();
@@ -71,6 +76,7 @@ public class PhantomArrayListCheckout implements AutoCloseable
public void addByteArrayList(ByteArrayList list) { this.byteArrayLists.add(list); }
public void addShortArrayList(ShortArrayList list) { this.shortArrayLists.add(list); }
public void addLongArrayListRef(LongArrayList list) { this.longArrayLists.add(list); }
+ public void addCharArrayListRef(CharArrayList list) { this.charArrayLists.add(list); }
@@ -81,6 +87,7 @@ public class PhantomArrayListCheckout implements AutoCloseable
public int getByteArrayCount() { return this.byteArrayLists.size(); }
public int getShortArrayCount() { return this.shortArrayLists.size(); }
public int getLongArrayCount() { return this.longArrayLists.size(); }
+ public int getCharArrayCount() { return this.charArrayLists.size(); }
@@ -102,10 +109,17 @@ public class PhantomArrayListCheckout implements AutoCloseable
ListUtil.clearAndSetSize(list, size);
return list;
}
+ public CharArrayList getCharArray(int index, int size)
+ {
+ CharArrayList list = this.charArrayLists.get(index);
+ ListUtil.clearAndSetSize(list, size);
+ return list;
+ }
public ArrayList getAllByteArrays() { return this.byteArrayLists; }
public ArrayList getAllShortArrays() { return this.shortArrayLists; }
public ArrayList getAllLongArrays() { return this.longArrayLists; }
+ public ArrayList getAllCharArrays() { return this.charArrayLists; }
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java
index 466dbf1ca..3f244e5c3 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java
@@ -11,6 +11,7 @@ import com.seibel.distanthorizons.core.util.objects.Pair;
import com.seibel.distanthorizons.coreapi.ModInfo;
import com.seibel.distanthorizons.coreapi.util.StringUtil;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
+import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.jetbrains.annotations.NotNull;
@@ -97,6 +98,11 @@ public class PhantomArrayListPool
private final AtomicInteger totalShortArrayCountRef = new AtomicInteger(0);
/** counts how many long arrays have been created by this pool */
private final AtomicInteger totalLongArrayCountRef = new AtomicInteger(0);
+ /** counts how many char arrays have been created by this pool */
+ private final AtomicInteger totalCharArrayCountRef = new AtomicInteger(0);
+ // NOTE: if we ever need to add another pool type we should separate out the logic for each individually pooled object.
+ // Otherwise, this class will just get too cluttered with duplicate code.
+ // But for now since this works, it'll be fine.
/** used for debugging, represents an estimate for how many bytes the byte[] pool contains */
private long lastBytePoolSizeInBytes = -1;
@@ -104,6 +110,8 @@ public class PhantomArrayListPool
private long lastShortPoolSizeInBytes = -1;
/** used for debugging, represents an estimate for how many bytes the long[] pool contains */
private long lastLongPoolSizeInBytes = -1;
+ /** used for debugging, represents an estimate for how many bytes the char[] pool contains */
+ private long lastCharPoolSizeInBytes = -1;
/** used for debugging, represents an estimate for how many byte[]'s are currently in this pool*/
private int lastBytePoolCount = 0;
@@ -111,8 +119,8 @@ public class PhantomArrayListPool
private int lastShortPoolCount = 0;
/** used for debugging, represents an estimate for how many long[]'s are currently in this pool*/
private int lastLongPoolCount = 0;
- /** used for debugging, represents an estimate for how many checkouts are currently in this pool*/
- private int lastCheckoutPoolCount = 0;
+ /** used for debugging, represents an estimate for how many char[]'s are currently in this pool*/
+ private int lastCharPoolCount = 0;
/** For pools backed by {@link SoftReference}'s we may need to decrease the size when elements are garbage collected */
private boolean clearLastRefPoolSizes = false;
@@ -144,12 +152,17 @@ public class PhantomArrayListPool
// get checkout //
//==============//
+ public PhantomArrayListCheckout checkoutByteArrays(int count) { return this.checkoutArrays(count, 0, 0, 0); }
+ public PhantomArrayListCheckout checkoutShortArrays(int count) { return this.checkoutArrays(0, count, 0, 0); }
+ public PhantomArrayListCheckout checkoutLongArrays(int count) { return this.checkoutArrays(0, 0, count, 0); }
+ public PhantomArrayListCheckout checkoutCharArrays(int count) { return this.checkoutArrays(0, 0, 0, count); }
+
/**
* If possible all checkouts for a given pool should be the same size,
* since {@link PhantomArrayListCheckout}'s are shared, returning the same size
* prevents accidentally returning a larger checkout than necessary, which wastes memory.
*/
- public PhantomArrayListCheckout checkoutArrays(int byteArrayCount, int shortArrayCount, int longArrayCount)
+ public PhantomArrayListCheckout checkoutArrays(int byteArrayCount, int shortArrayCount, int longArrayCount, int charArrayCount)
{
PhantomArrayListCheckout checkout = null;
while (checkout == null)
@@ -219,6 +232,12 @@ public class PhantomArrayListPool
checkout.addLongArrayListRef(this.createEmptyLongArrayList());
}
+ // char
+ for (int i = checkout.getCharArrayCount(); i < charArrayCount; i++)
+ {
+ checkout.addCharArrayListRef(this.createEmptyCharArrayList());
+ }
+
return checkout;
}
@@ -243,12 +262,19 @@ public class PhantomArrayListPool
this.totalLongArrayCountRef.getAndIncrement();
return new LongArrayList(0);
}
+ private CharArrayList createEmptyCharArrayList()
+ {
+ //LOGGER.error("created new char array");
+ this.totalCharArrayCountRef.getAndIncrement();
+ return new CharArrayList(0);
+ }
//==================//
// phantom recovery //
//==================//
+ ///region
private static void runPhantomReferenceCleanupLoop()
{
@@ -273,6 +299,7 @@ public class PhantomArrayListPool
int returnedByteArrayCount = 0;
int returnedShortArrayCount = 0;
int returnedLongArrayCount = 0;
+ int returnedCharArrayCount = 0;
int checkoutCount = 0;
allocationStackTraceCountPairList.clear();
@@ -287,6 +314,7 @@ public class PhantomArrayListPool
returnedByteArrayCount += checkout.getByteArrayCount();
returnedShortArrayCount += checkout.getShortArrayCount();
returnedLongArrayCount += checkout.getLongArrayCount();
+ returnedCharArrayCount += checkout.getCharArrayCount();
checkoutCount++;
pool.returnCheckout(checkout);
@@ -311,9 +339,16 @@ public class PhantomArrayListPool
if (checkoutCount != 0
|| returnedByteArrayCount != 0
|| returnedShortArrayCount != 0
- || returnedLongArrayCount != 0)
+ || returnedLongArrayCount != 0
+ || returnedCharArrayCount != 0)
{
- LOGGER.warn("Pool: ["+ pool.name+"] phantom recovery. Returned checkouts:["+F3Screen.NUMBER_FORMAT.format(checkoutCount)+"], byte:["+F3Screen.NUMBER_FORMAT.format(returnedByteArrayCount)+"], short:["+F3Screen.NUMBER_FORMAT.format(returnedShortArrayCount)+"], long:["+F3Screen.NUMBER_FORMAT.format(returnedLongArrayCount)+"].");
+ LOGGER.warn("Pool: ["+ pool.name+"] phantom recovery. " +
+ "Returned checkouts:["+F3Screen.NUMBER_FORMAT.format(checkoutCount)+"], " +
+ "byte:["+F3Screen.NUMBER_FORMAT.format(returnedByteArrayCount)+"], " +
+ "short:["+F3Screen.NUMBER_FORMAT.format(returnedShortArrayCount)+"], " +
+ "long:["+F3Screen.NUMBER_FORMAT.format(returnedLongArrayCount)+"]," +
+ "char:["+F3Screen.NUMBER_FORMAT.format(returnedCharArrayCount)+"]."
+ );
// log stack traces if present
if (pool.logGarbageCollectedStacks)
@@ -375,11 +410,14 @@ public class PhantomArrayListPool
}
}
+ ///endregion
+
//=================//
// return checkout //
//=================//
+ ///region
public void returnParentPhantomRef(@NotNull PhantomReference parentRef)
{
@@ -407,18 +445,21 @@ public class PhantomArrayListPool
//LOGGER.info("Returned ["+checkout.byteArrayLists.size()+"/"+this.pooledByteArrays.size()+"] bytes and ["+checkout.longArrayLists.size()+"/"+this.pooledLongArrays.size()+"] longs.");\
}
-
+
+ ///endregion
+
//===============//
// debug methods //
//===============//
+ ///region
public static void addDebugMenuStringsToListForCombinedPools(List messageList)
{
- int totalByteArrayCount = 0, totalShortArrayCount = 0, totalLongArrayCount = 0;
- int pooledByteArraySize = 0, pooledShortArraySize = 0, pooledLongArraySize = 0;
- long lastBytePoolSizeInBytes = 0, lastShortPoolSizeInBytes = 0, lastLongPoolSizeInBytes = 0;
+ int totalByteArrayCount = 0, totalShortArrayCount = 0, totalLongArrayCount = 0, totalCharArrayCount = 0;
+ int pooledByteArraySize = 0, pooledShortArraySize = 0, pooledLongArraySize = 0, pooledCharArraySize = 0;
+ long lastBytePoolSizeInBytes = 0, lastShortPoolSizeInBytes = 0, lastLongPoolSizeInBytes = 0, lastCharPoolSizeInBytes = 0;
for (int i = 0; i < POOL_LIST.size(); i++)
{
@@ -427,21 +468,24 @@ public class PhantomArrayListPool
totalByteArrayCount += pool.totalByteArrayCountRef.get();
totalShortArrayCount += pool.totalShortArrayCountRef.get();
totalLongArrayCount += pool.totalLongArrayCountRef.get();
+ totalCharArrayCount += pool.totalCharArrayCountRef.get();
pooledByteArraySize += pool.lastBytePoolCount;
pooledShortArraySize += pool.lastShortPoolCount;
pooledLongArraySize += pool.lastLongPoolCount;
+ pooledCharArraySize += pool.lastCharPoolCount;
lastBytePoolSizeInBytes += pool.lastBytePoolSizeInBytes;
lastShortPoolSizeInBytes += pool.lastShortPoolSizeInBytes;
lastLongPoolSizeInBytes += pool.lastLongPoolSizeInBytes;
+ lastCharPoolSizeInBytes += pool.lastCharPoolSizeInBytes;
}
addDebugMenuStringsToList(messageList,
"Combined",
- totalByteArrayCount, totalShortArrayCount, totalLongArrayCount,
- pooledByteArraySize, pooledShortArraySize, pooledLongArraySize,
- lastBytePoolSizeInBytes, lastShortPoolSizeInBytes, lastLongPoolSizeInBytes
+ totalByteArrayCount, totalShortArrayCount, totalLongArrayCount, totalCharArrayCount,
+ pooledByteArraySize, pooledShortArraySize, pooledLongArraySize, pooledCharArraySize,
+ lastBytePoolSizeInBytes, lastShortPoolSizeInBytes, lastLongPoolSizeInBytes, lastCharPoolSizeInBytes
);
}
@@ -458,26 +502,28 @@ public class PhantomArrayListPool
{
addDebugMenuStringsToList(messageList,
this.name,
- this.totalByteArrayCountRef.get(), this.totalShortArrayCountRef.get(), this.totalLongArrayCountRef.get(),
- this.lastBytePoolCount, this.lastShortPoolCount, this.lastLongPoolCount,
- this.lastBytePoolSizeInBytes, this.lastShortPoolSizeInBytes, this.lastLongPoolSizeInBytes
+ this.totalByteArrayCountRef.get(), this.totalShortArrayCountRef.get(), this.totalLongArrayCountRef.get(), this.totalCharArrayCountRef.get(),
+ this.lastBytePoolCount, this.lastShortPoolCount, this.lastLongPoolCount, this.lastCharPoolCount,
+ this.lastBytePoolSizeInBytes, this.lastShortPoolSizeInBytes, this.lastLongPoolSizeInBytes, this.lastCharPoolSizeInBytes
);
}
private static void addDebugMenuStringsToList(List messageList,
String name,
- int totalByteArrayCount, int totalShortArrayCount, int totalLongArrayCount,
- int numbOfByteArraysInPool, int numbOfShortArraysInPool, int numbOfLongArraysInPool,
- long lastBytePoolSizeInBytes, long lastShortPoolSizeInBytes, long lastLongPoolSizeInBytes)
+ int totalByteArrayCount, int totalShortArrayCount, int totalLongArrayCount, int totalCharArrayCount,
+ int numbOfByteArraysInPool, int numbOfShortArraysInPool, int numbOfLongArraysInPool, int numbOfCharArraysInPool,
+ long lastBytePoolSizeInBytes, long lastShortPoolSizeInBytes, long lastLongPoolSizeInBytes, long lastCharPoolSizeInBytes)
{
// total (all time created) count
String byteArrayTotalCount = F3Screen.NUMBER_FORMAT.format(totalByteArrayCount);
String shortArrayTotalCount = F3Screen.NUMBER_FORMAT.format(totalShortArrayCount);
String longArrayTotalCount = F3Screen.NUMBER_FORMAT.format(totalLongArrayCount);
+ String charArrayTotalCount = F3Screen.NUMBER_FORMAT.format(totalCharArrayCount);
// inactive items in pool
String bytePoolCount = F3Screen.NUMBER_FORMAT.format(numbOfByteArraysInPool);
String shortPoolCount = F3Screen.NUMBER_FORMAT.format(numbOfShortArraysInPool);
String longPoolCount = F3Screen.NUMBER_FORMAT.format(numbOfLongArraysInPool);
+ String charPoolCount = F3Screen.NUMBER_FORMAT.format(numbOfCharArraysInPool);
// pool byte size
String bytePoolSizeInBytes = (lastBytePoolSizeInBytes != -1)
@@ -489,6 +535,9 @@ public class PhantomArrayListPool
String longPoolSizeInBytes = (lastLongPoolSizeInBytes != -1)
? " ~" + StringUtil.convertBytesToHumanReadable(lastLongPoolSizeInBytes)
: "";
+ String charPoolSizeInBytes = (lastCharPoolSizeInBytes != -1)
+ ? " ~" + StringUtil.convertBytesToHumanReadable(lastCharPoolSizeInBytes)
+ : "";
messageList.add(name + " - Pools:");
@@ -504,6 +553,10 @@ public class PhantomArrayListPool
{
messageList.add("long[]: " + longPoolCount + "/" + longArrayTotalCount + longPoolSizeInBytes);
}
+ if (totalCharArrayCount != 0)
+ {
+ messageList.add("char[]: " + charPoolCount + "/" + charArrayTotalCount + charPoolSizeInBytes);
+ }
}
@@ -516,10 +569,12 @@ public class PhantomArrayListPool
long bytePoolByteSize = 0;
long shortPoolByteSize = 0;
long longPoolByteSize = 0;
+ long charPoolByteSize = 0;
int bytePoolCount = 0;
int shortPoolCount = 0;
int longPoolCount = 0;
+ int charPoolCount = 0;
// checkouts //
@@ -537,6 +592,8 @@ public class PhantomArrayListPool
shortPoolCount += pooledCheckout.getAllShortArrays().size();
longPoolByteSize += estimateMemoryUsage(pooledCheckout.getAllLongArrays(), Long.BYTES);
longPoolCount += pooledCheckout.getAllLongArrays().size();
+ charPoolByteSize += estimateMemoryUsage(pooledCheckout.getAllCharArrays(), Character.BYTES);
+ charPoolCount += pooledCheckout.getAllCharArrays().size();
}
@@ -546,11 +603,10 @@ public class PhantomArrayListPool
this.lastBytePoolSizeInBytes = 0;
this.lastShortPoolSizeInBytes = 0;
this.lastLongPoolSizeInBytes = 0;
+ this.lastCharPoolSizeInBytes = 0;
this.clearLastRefPoolSizes = false;
}
- this.lastCheckoutPoolCount = this.pooledCheckoutsRefs.size();
-
// byte //
// math.max is used since the pool should only grow until a soft reference is freed,
// and it's easier to understand if this constantly grows instead of jumping around
@@ -564,6 +620,10 @@ public class PhantomArrayListPool
// long //
this.lastLongPoolSizeInBytes = Math.max(longPoolByteSize, this.lastLongPoolSizeInBytes);
this.lastLongPoolCount = longPoolCount;
+
+ // char //
+ this.lastCharPoolSizeInBytes = Math.max(charPoolByteSize, this.lastCharPoolSizeInBytes);
+ this.lastCharPoolCount = charPoolCount;
}
private static > long estimateMemoryUsage(Iterable pool, long elementSizeInBytes)
@@ -614,6 +674,10 @@ public class PhantomArrayListPool
{
elementCount = ((LongArrayList)array).elements().length;
}
+ else if (array instanceof CharArrayList)
+ {
+ elementCount = ((CharArrayList)array).elements().length;
+ }
else
{
throw new UnsupportedOperationException("Not implemented for type ["+array.getClass().getSimpleName()+"].");
@@ -622,6 +686,8 @@ public class PhantomArrayListPool
return elementCount;
}
+ ///endregion
+
}
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/StringPool.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/StringPool.java
new file mode 100644
index 000000000..a291ade70
--- /dev/null
+++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/StringPool.java
@@ -0,0 +1,292 @@
+package com.seibel.distanthorizons.core.pooling;
+
+import com.seibel.distanthorizons.core.logging.DhLogger;
+import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import it.unimi.dsi.fastutil.chars.CharArrayList;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.StampedLock;
+
+/**
+ * A thread-safe string pool backed by a trie.
+ *
+ * @link https://en.wikipedia.org/wiki/Trie
+ * @link https://claude.ai/share/eb7fddbe-03a0-4562-88a0-a089b7a52006
+ */
+public class StringPool
+{
+ private static final DhLogger LOGGER = new DhLoggerBuilder().build();
+ public static final StringPool INSTANCE = new StringPool();
+
+ private final TrieNode root = new TrieNode();
+
+
+
+ //=============//
+ // constructor //
+ //=============//
+ ///region
+
+ // for now we only need a single INSTANCE,
+ // if that changes in the future we can change this up
+ private StringPool() {}
+
+ ///endregion
+
+
+
+ //===================//
+ // get pooled string //
+ //===================//
+ ///region
+
+ /**
+ * Returns a pooled String instance for the given character array.
+ * If an equivalent string already exists in the pool, returns that instance.
+ * Otherwise, creates a new String and adds it to the pool.
+ *
+ * @param chars the character array to convert to a pooled string
+ * @return a pooled String instance
+ */
+ public String getPooledString(CharArrayList chars) { return this.getPooledString(chars, 0, chars.size()); }
+
+ /**
+ * Returns a pooled String instance for a substring of the given character array.
+ *
+ * @param chars the character array
+ * @param offset the starting offset
+ * @param length the number of characters to use
+ * @return a pooled String instance
+ */
+ public String getPooledString(CharArrayList chars, int offset, int length)
+ {
+ if (length == 0)
+ {
+ return "";
+ }
+
+ TrieNode currentNode = this.root;
+
+ // Navigate/create the trie path
+ for (int i = 0; i < length; i++)
+ {
+ char c = chars.getChar(offset + i);
+ currentNode = currentNode.getOrCreateChild(c);
+ }
+
+ // Get or set the string at the leaf node
+ return currentNode.getOrSetString(chars, offset, length);
+ }
+
+ ///endregion
+
+
+
+ //================//
+ // helper methods //
+ //================//
+ ///region
+
+ public void clear() { this.root.clear(); }
+
+ /**
+ * Returns an approximate count of pooled strings.
+ * Note: This is a best-effort count and may not be exact in concurrent scenarios.
+ */
+ public long approximateSize() { return this.root.countStrings(); }
+
+ ///endregion
+
+
+
+ //================//
+ // helper classes //
+ //================//
+ ///region
+
+ /**
+ * Node in the trie structure. Uses a {@link StampedLock} for efficient concurrent access
+ * with optimistic reads for the common use case where the node already exists.
+ */
+ private static class TrieNode
+ {
+ private volatile ConcurrentHashMap children;
+ private volatile String value;
+ private final StampedLock lock = new StampedLock();
+
+
+
+ //================//
+ // node interface //
+ //================//
+ ///region
+
+ TrieNode getOrCreateChild(char inputChar)
+ {
+ // Optimistic read - try without locking first
+ long stamp = this.lock.tryOptimisticRead();
+ ConcurrentHashMap currentChildren = this.children;
+
+ // check for an existing child node
+ if (stamp != 0 // zero means exclusively locked
+ && this.lock.validate(stamp)
+ && currentChildren != null)
+ {
+ TrieNode child = currentChildren.get(inputChar);
+ if (child != null)
+ {
+ return child;
+ }
+ }
+
+ // we need to acquire a read lock to safely check again
+ stamp = this.lock.readLock();
+ try
+ {
+ if (this.children != null)
+ {
+ TrieNode child = this.children.get(inputChar);
+ if (child != null)
+ {
+ return child;
+ }
+ }
+
+ // Upgrade to write lock to create the child
+ long writeStamp = this.lock.tryConvertToWriteLock(stamp);
+ if (writeStamp == 0)
+ {
+ this.lock.unlockRead(stamp);
+ writeStamp = this.lock.writeLock();
+ }
+ stamp = writeStamp;
+
+ // add the missing node if needed (check in case of concurrency)
+ if (this.children == null)
+ {
+ this.children = new ConcurrentHashMap<>();
+ }
+ return this.children.computeIfAbsent(inputChar, newChar -> new TrieNode());
+ }
+ finally
+ {
+ this.lock.unlock(stamp);
+ }
+ }
+
+ String getOrSetString(CharArrayList chars, int offset, int length)
+ {
+ // Optimistic read
+ long stamp = this.lock.tryOptimisticRead();
+ String currentValue = this.value;
+
+ if (stamp != 0
+ && this.lock.validate(stamp)
+ && currentValue != null)
+ {
+ return currentValue;
+ }
+
+ // Acquire read lock
+ stamp = this.lock.readLock();
+ try
+ {
+ if (this.value != null)
+ {
+ return this.value;
+ }
+
+ // Upgrade to write lock
+ long writeStamp = this.lock.tryConvertToWriteLock(stamp);
+ if (writeStamp == 0)
+ {
+ // already locked by another thread, wait
+ this.lock.unlockRead(stamp);
+ writeStamp = this.lock.writeLock();
+ }
+ stamp = writeStamp;
+
+ // Double-check
+ if (this.value != null)
+ {
+ return this.value;
+ }
+
+ // create our newly pooled string
+ this.value = new String(chars.elements(), offset, length);
+ return this.value;
+ }
+ finally
+ {
+ this.lock.unlock(stamp);
+ }
+ }
+
+ ///endregion
+
+
+
+ //================//
+ // helper methods //
+ //================//
+ ///region
+
+ void clear()
+ {
+ long stamp = this.lock.writeLock();
+ try
+ {
+ if (this.children != null)
+ {
+ this.children.clear();
+ }
+ this.children = null;
+ this.value = null;
+ }
+ finally
+ {
+ this.lock.unlock(stamp);
+ }
+ }
+
+ long countStrings()
+ {
+ long stamp = this.lock.tryOptimisticRead();
+ ConcurrentHashMap currentChildren = this.children;
+ String currentValue = this.value;
+
+ if (!this.lock.validate(stamp))
+ {
+ stamp = this.lock.readLock();
+ try
+ {
+ currentChildren = this.children;
+ currentValue = this.value;
+ }
+ finally
+ {
+ this.lock.unlockRead(stamp);
+ }
+ }
+
+ long count = currentValue != null ? 1 : 0;
+ if (currentChildren != null)
+ {
+ for (TrieNode child : currentChildren.values())
+ {
+ count += child.countStrings();
+ }
+ }
+ return count;
+ }
+
+ ///endregion
+
+ }
+
+ ///endregion
+
+
+
+}
+
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/QuadTree/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/QuadTree/LodQuadTree.java
index b06899d76..8cff8e3a2 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/render/QuadTree/LodQuadTree.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/render/QuadTree/LodQuadTree.java
@@ -300,7 +300,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen
}
// queue full data retrieval (world gen) requests if needed
- if (this.missingGenerationPosSet.size() != 0 // TODO can stay empty if generation is toggled at the wrong time (IE world gen starts, turn it off, then turn it back on)
+ if (this.missingGenerationPosSet.size() != 0
&& this.fullDataSourceProvider.canQueueRetrievalNow()
&& !this.queueThreadRunningRef.get())
{
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java
index 81c886a23..572407345 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java
@@ -66,14 +66,6 @@ public class FullDataSourceV1DTO implements IBaseDTO
}
- /** @return a stream for the data contained in this DTO. */
- public DhDataInputStream getInputStream() throws IOException
- {
- DhDataInputStream compressedStream = DhDataInputStream.create(this.dataArray, EDhApiDataCompressionMode.LZ4); // LZ4 was used by DH before 2.1.0 and as such must be used until the render data format is changed to record the compressor
- return compressedStream;
- }
-
-
//===========//
// overrides //
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
index 71736f2dc..8a75e6799 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
@@ -28,6 +28,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.network.INetworkObject;
@@ -56,6 +57,8 @@ public class FullDataSourceV2DTO
{
public static final boolean VALIDATE_INPUT_DATAPOINTS = true;
+ public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("V2DTO");
+
public static class DATA_FORMAT
{
public static final int V1_NO_ADJACENT_DATA = 1;
@@ -96,9 +99,6 @@ public class FullDataSourceV2DTO
public long createdUnixDateTime;
- public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("V2DTO");
-
-
//==============//
// constructors //
@@ -140,7 +140,7 @@ public class FullDataSourceV2DTO
public static FullDataSourceV2DTO CreateEmptyDataSourceForDecoding() { return new FullDataSourceV2DTO(); }
private FullDataSourceV2DTO()
{
- super(ARRAY_LIST_POOL, 8, 0, 0);
+ super(ARRAY_LIST_POOL, 8, 0, 0, 0);
// Expected sizes here are 0 since we don't know how big these arrays need to be,
// they depend on compression settings and world complexity.
@@ -278,13 +278,7 @@ public class FullDataSourceV2DTO
throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests.");
}
- FullDataPointIdMap newMap = readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, compressionModeEnum);
- dataSource.mapping.addAll(newMap);
- if (dataSource.mapping.size() != newMap.size())
- {
- // if the mappings are out of sync then the LODs will render incorrectly due to IDs being wrong
- LodUtil.assertNotReach("ID maps out of sync for pos: "+this.pos);
- }
+ readBlobToDataMapping(dataSource.mapping, this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, compressionModeEnum);
}
@@ -344,7 +338,8 @@ public class FullDataSourceV2DTO
ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray,
EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
- try (DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum))
+ try (PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1);
+ DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum, checkout))
{
// read the data
int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH;
@@ -540,7 +535,8 @@ public class FullDataSourceV2DTO
}
- try (DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum))
+ try (PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1);
+ DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum, checkout))
{
// 1. column counts, preallocate
for (int x = minX; x < maxX; x++)
@@ -676,7 +672,8 @@ public class FullDataSourceV2DTO
}
private static void readBlobToGenerationSteps(ByteArrayList inputCompressedDataByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
- try(DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum))
+ try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1);
+ DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum, checkout))
{
compressedIn.readFully(outputByteArray.elements(), 0, FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH);
}
@@ -699,7 +696,8 @@ public class FullDataSourceV2DTO
}
private static void readBlobToWorldCompressionMode(ByteArrayList inputCompressedDataByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
- try(DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum))
+ try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1);
+ DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum, checkout))
{
compressedIn.readFully(outputByteArray.elements(), 0, FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH);
}
@@ -717,12 +715,12 @@ public class FullDataSourceV2DTO
mapping.serialize(compressedOut);
}
}
- private static FullDataPointIdMap readBlobToDataMapping(ByteArrayList inputCompressedDataByteArray, long pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException
+ private static void readBlobToDataMapping(FullDataPointIdMap mapping, ByteArrayList inputCompressedDataByteArray, long pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException
{
- try (DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum))
+ try (PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutByteArrays(1);
+ DhDataInputStream compressedIn = DhDataInputStream.create(inputCompressedDataByteArray, compressionModeEnum, checkout))
{
- FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper);
- return mapping;
+ FullDataPointIdMap.deserialize(mapping, compressedIn, pos, levelWrapper);
}
}
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
index 92194233f..17af8af33 100644
--- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
+++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
@@ -24,6 +24,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.DbConnectionClosedException;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
@@ -576,7 +577,8 @@ public class FullDataSourceV2Repo extends AbstractDhRepo