diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 30dbd7820..cf08e3bf0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -382,7 +382,7 @@ public class FullDataSourceV1 implements IDataSource //==================// @Override - public void close() throws Exception + public void close() { /* not currently needed */ } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 88ec4119c..263036073 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -26,18 +26,16 @@ import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; -import com.seibel.distanthorizons.core.file.DataSourcePool; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.DhApiTerrainDataPointUtil; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.RenderDataPointUtil; +import com.seibel.distanthorizons.core.util.*; import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -53,7 +51,9 @@ import java.util.List; * @see FullDataPointUtil * @see FullDataSourceV1 */ -public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSource +public class FullDataSourceV2 + extends PhantomArrayListParent + implements IDataSource, IDhApiFullDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ @@ -72,13 +72,11 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo public static final byte DATA_FORMAT_VERSION = 1; - public static final DataSourcePool DATA_SOURCE_POOL = new DataSourcePool<>(FullDataSourceV2::createEmpty, FullDataSourceV2::prepPooledDataSource); - private int cachedHashCode = 0; - private long pos; + private final long pos; @Override public Long getKey() { return this.pos; } @Override @@ -98,13 +96,13 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo * * @see EDhApiWorldGenerationStep */ - public byte[] columnGenerationSteps; + public final ByteArrayList columnGenerationSteps; /** * stores what world compression was used for each column. * * @see EDhApiWorldCompressionMode */ - public byte[] columnWorldCompressionMode; + public final ByteArrayList columnWorldCompressionMode; /** * stored x/z, y
@@ -112,12 +110,12 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo * TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately * there's something in the render data logic that expects this order so we can't change it right now */ - public LongArrayList[] dataPoints; + public final LongArrayList[] dataPoints; public boolean isEmpty; public boolean applyToParent = false; - /** should only be used by methods exposed by the DH API */ + /** should only be used by methods exposed via the DH API */ private boolean runApiChunkValidation = false; @@ -126,34 +124,6 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo // constructors // //==============// - public static FullDataSourceV2 createEmpty(long pos) { return new FullDataSourceV2(pos); } - private FullDataSourceV2(long pos) - { - this.pos = pos; - this.dataPoints = new LongArrayList[WIDTH * WIDTH]; - this.mapping = new FullDataPointIdMap(pos); - this.isEmpty = true; - - // doesn't need to be populated since nothing has been generated yet - // the default value of all 0's is adequate - this.columnGenerationSteps = new byte[WIDTH * WIDTH]; - this.columnWorldCompressionMode = new byte[WIDTH * WIDTH]; - } - - public static FullDataSourceV2 createWithData(long pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep, byte[] columnWorldCompressionMode) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep, columnWorldCompressionMode); } - private FullDataSourceV2(long pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps, byte[] columnWorldCompressionMode) - { - LodUtil.assertTrue(data.length == WIDTH * WIDTH); - - this.pos = pos; - this.dataPoints = data; - this.mapping = mapping; - this.isEmpty = false; - - this.columnGenerationSteps = columnGenerationSteps; - this.columnWorldCompressionMode = columnWorldCompressionMode; - } - public static FullDataSourceV2 createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createFromChunk(chunkWrapper); } public static FullDataSourceV2 createFromLegacyDataSourceV1(FullDataSourceV1 legacyData) @@ -225,6 +195,84 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo return fullDataSource; } + public static FullDataSourceV2 createEmpty(long pos) + { + return new FullDataSourceV2( + pos, new FullDataPointIdMap(pos), + // data points, genSteps, and columnCompression are all null since + // nothing has been generated yet. + // Using the default value of all 0's is adequate + null, + null, null, + true); + } + + public static FullDataSourceV2 createWithData(long pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep, byte[] columnWorldCompressionMode) + { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep, columnWorldCompressionMode, false); } + + private FullDataSourceV2( + long pos, + FullDataPointIdMap mapping, @Nullable LongArrayList[] data, + @Nullable byte[] columnGenerationSteps, @Nullable byte[] columnWorldCompressionMode, + boolean empty) + { + super(2, 0, WIDTH * WIDTH); + + LodUtil.assertTrue(data == null || data.length == WIDTH * WIDTH); + + + + this.pos = pos; + this.mapping = mapping; + this.isEmpty = empty; + + + // pooled data arrays + this.dataPoints = new LongArrayList[WIDTH * WIDTH]; + for (int i = 0; i < WIDTH * WIDTH; i++) + { + // size defaulting to 0 since we don't know how many datapoints + // will be in this column yet + this.dataPoints[i] = this.pooledArraysCheckout.getLongArray(i, 0); + } + + // use incoming data if present + if (data != null) + { + for (int i = 0; i < WIDTH * WIDTH; i++) + { + this.dataPoints[i].addAll(data[i]); + } + } + + // pooled generation step array + this.columnGenerationSteps = this.pooledArraysCheckout.getByteArray(0); + if (columnGenerationSteps != null) + { + this.columnGenerationSteps.addElements(0, columnGenerationSteps); + } + else + { + ListUtil.clearAndSetSize(this.columnGenerationSteps, WIDTH * WIDTH); + } + + // pooled column compression array + this.columnWorldCompressionMode = this.pooledArraysCheckout.getByteArray(1); + if (columnWorldCompressionMode != null) + { + this.columnWorldCompressionMode.addElements(0, columnWorldCompressionMode); + } + else + { + ListUtil.clearAndSetSize(this.columnWorldCompressionMode, WIDTH * WIDTH); + } + + + // the pooled arrays have all been set, + // the checkout object is no longer needed + this.pooledArraysCheckout = null; + } + //======// @@ -300,8 +348,8 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo LongArrayList inputDataArray = inputDataSource.dataPoints[index]; if (inputDataArray != null) { - byte thisGenState = this.columnGenerationSteps[index]; - byte inputGenState = inputDataSource.columnGenerationSteps[index]; + byte thisGenState = this.columnGenerationSteps.getByte(index); + byte inputGenState = inputDataSource.columnGenerationSteps.getByte(index); if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value && thisGenState <= inputGenState) @@ -352,9 +400,9 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo } - this.columnGenerationSteps[index] = inputGenState; + this.columnGenerationSteps.set(index, inputGenState); // always overwrite the compression mode since we're replacing this column - this.columnWorldCompressionMode[index] = inputDataSource.columnWorldCompressionMode[index]; + this.columnWorldCompressionMode.set(index, inputDataSource.columnWorldCompressionMode.getByte(index)); this.isEmpty = false; } } @@ -397,12 +445,12 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo // world gen // byte inputGenStep = determineMinWorldGenStepForTwoByTwoColumn(inputDataSource.columnGenerationSteps, x, z); - this.columnGenerationSteps[recipientIndex] = inputGenStep; + this.columnGenerationSteps.set(recipientIndex, inputGenStep); // world compression // byte worldCompressionMode = determineHighestWorldCompressionForTwoByTwoColumn(inputDataSource.columnWorldCompressionMode, x, z); - this.columnWorldCompressionMode[recipientIndex] = worldCompressionMode; + this.columnWorldCompressionMode.set(recipientIndex, worldCompressionMode); @@ -461,7 +509,7 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo * The minimum value is used because we don't want to accidentally record that * something was generated when it wasn't. */ - private static byte determineMinWorldGenStepForTwoByTwoColumn(byte[] columnGenerationSteps, int relX, int relZ) + private static byte determineMinWorldGenStepForTwoByTwoColumn(ByteArrayList columnGenerationSteps, int relX, int relZ) { // TODO merge similar logic with determineHighestWorldCompressionForTwoByTwoColumn byte minWorldGenStepValue = Byte.MAX_VALUE; @@ -470,7 +518,7 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo for (int z = 0; z < 2; z++) { int index = relativePosToIndex(x + relX, z + relZ); - byte worldGenStepValue = columnGenerationSteps[index]; + byte worldGenStepValue = columnGenerationSteps.getByte(index); minWorldGenStepValue = (byte) Math.min(minWorldGenStepValue, worldGenStepValue); } } @@ -480,7 +528,7 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo * The minimum value is used because we don't want to accidentally record that * something was generated when it wasn't. */ - private static byte determineHighestWorldCompressionForTwoByTwoColumn(byte[] columnCompressionMode, int relX, int relZ) + private static byte determineHighestWorldCompressionForTwoByTwoColumn(ByteArrayList columnCompressionMode, int relX, int relZ) { // TODO merge similar logic with determineMinWorldGenStepForTwoByTwoColumn byte minWorldGenStepValue = Byte.MIN_VALUE; @@ -489,7 +537,7 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo for (int z = 0; z < 2; z++) { int index = relativePosToIndex(x + relX, z + relZ); - byte worldGenStepValue = columnCompressionMode[index]; + byte worldGenStepValue = columnCompressionMode.getByte(index); minWorldGenStepValue = (byte) Math.max(minWorldGenStepValue, worldGenStepValue); } } @@ -855,37 +903,6 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo - //=========// - // pooling // - //=========// - - private static void prepPooledDataSource(long pos, boolean clearData, FullDataSourceV2 dataSource) - { - dataSource.pos = pos; - - if (clearData) - { - dataSource.mapping.clear(pos); - - for (int i = 0; i < dataSource.dataPoints.length; i++) - { - if (dataSource.dataPoints[i] != null) - { - dataSource.dataPoints[i].clear(); - } - } - - Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); - Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0); - } - - // default to API validation disabled so it's opt-in - // to reduce the chance of performance loss with unnecessary validation - dataSource.setRunApiChunkValidation(false); - } - - - //=====================// // setters and getters // //=====================// @@ -899,15 +916,15 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { int index = relativePosToIndex(relX, relZ); - return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); + return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps.getByte(index)); } public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep, EDhApiWorldCompressionMode worldCompressionMode) { int index = relativePosToIndex(relX, relZ); this.dataPoints[index] = longArray; - this.columnGenerationSteps[index] = worldGenStep.value; - this.columnWorldCompressionMode[index] = worldCompressionMode.value; + this.columnGenerationSteps.set(index, worldGenStep.value); + this.columnWorldCompressionMode.set(index, worldCompressionMode.value); if (RUN_UPDATE_DEV_VALIDATION) @@ -1002,8 +1019,8 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo { int result = DhSectionPos.hashCode(this.pos); result = 31 * result + Arrays.deepHashCode(this.dataPoints); - result = 17 * result + Arrays.hashCode(this.columnGenerationSteps); - result = 43 * result + Arrays.hashCode(this.columnWorldCompressionMode); + result = 17 * result + this.columnGenerationSteps.hashCode(); + result = 43 * result + this.columnWorldCompressionMode.hashCode(); this.cachedHashCode = result; } @@ -1030,8 +1047,6 @@ public class FullDataSourceV2 implements IDataSource, IDhApiFullDataSo } } - @Override - public void close() throws Exception - { DATA_SOURCE_POOL.returnPooledDataSource(this); } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 8c12157b5..d6d0aaefe 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -22,9 +22,9 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.DataSourcePool; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.coreapi.ModInfo; @@ -37,7 +37,6 @@ import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; -import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; /** @@ -45,7 +44,9 @@ import java.util.concurrent.atomic.AtomicLong; * * @see RenderDataPointUtil */ -public class ColumnRenderSource implements IDataSource +public class ColumnRenderSource + extends PhantomArrayListParent + implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -54,8 +55,6 @@ public class ColumnRenderSource implements IDataSource /** width of this data in columns */ public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); // 64 - public static final DataSourcePool DATA_SOURCE_POOL = new DataSourcePool<>(ColumnRenderSource::createEmptyRenderSource, null /* data source prep/cleanup needs to be done outside the pool since it requires additional inputs */); - /** will be zero if an empty data source was created */ @@ -77,40 +76,8 @@ public class ColumnRenderSource implements IDataSource // constructors // //==============// - /** - * This is separate from {@link DataSourcePool#getPooledSource(long, boolean)} - * because we need to pass in a couple extra values, - * specifically maxVerticalSize and yOffset. - */ - public static ColumnRenderSource getPooledRenderSource(long pos, int maxVerticalSize, int yOffset, boolean clearData) - { - ColumnRenderSource renderSource = DATA_SOURCE_POOL.getPooledSource(pos); - - // set necessary properties - renderSource.pos = pos; - renderSource.verticalDataCount = maxVerticalSize; - renderSource.yOffset = yOffset; - - - // resize the array if necessary - int dataArraySize = SECTION_SIZE * SECTION_SIZE * maxVerticalSize; - renderSource.renderDataContainer.ensureCapacity(dataArraySize); - while (renderSource.renderDataContainer.size() < dataArraySize) - { - renderSource.renderDataContainer.add(0); - } - - if (clearData) - { - Arrays.fill(renderSource.renderDataContainer.elements(), 0); - Arrays.fill(renderSource.debugSourceFlags, null); - } - - return renderSource; - } - - - private static ColumnRenderSource createEmptyRenderSource(long sectionPos) { return new ColumnRenderSource(sectionPos, 0, 0); } + public static ColumnRenderSource createEmpty(long pos, int maxVerticalSize, int yOffset) + { return new ColumnRenderSource(pos, maxVerticalSize, yOffset); } /** * Creates an empty ColumnRenderSource. * @@ -119,11 +86,16 @@ public class ColumnRenderSource implements IDataSource */ private ColumnRenderSource(long pos, int maxVerticalSize, int yOffset) { - this.verticalDataCount = maxVerticalSize; - this.renderDataContainer = new LongArrayList(new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]); - this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; + super(0, 0, 1); + this.pos = pos; this.yOffset = yOffset; + + this.verticalDataCount = maxVerticalSize; + + this.renderDataContainer = this.pooledArraysCheckout.getLongArray(0, SECTION_SIZE * SECTION_SIZE * this.verticalDataCount); + + this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; } @@ -327,12 +299,6 @@ public class ColumnRenderSource implements IDataSource return stringBuilder.toString(); } - @Override - public void close() throws Exception - { - DATA_SOURCE_POOL.returnPooledDataSource(this); - } - //==============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index a6b1558e1..6687e4635 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -28,13 +28,11 @@ import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArra import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPosMutable; -import com.seibel.distanthorizons.core.util.ColorUtil; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.RenderDataPointUtil; +import com.seibel.distanthorizons.core.util.*; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; @@ -43,6 +41,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; import java.util.HashSet; @@ -64,6 +63,7 @@ public class FullDataToRenderDataTransformer // public transformer interface // //==============================// + @Nullable public static ColumnRenderSource transformFullDataToRenderSource(FullDataSourceV2 fullDataSource, IDhClientLevel level) { if (fullDataSource == null) @@ -109,7 +109,7 @@ public class FullDataToRenderDataTransformer - final ColumnRenderSource columnSource = ColumnRenderSource.getPooledRenderSource(pos, vertSize, level.getMinY(), true); + final ColumnRenderSource columnSource = ColumnRenderSource.createEmpty(pos, vertSize, level.getMinY()); if (fullDataSource.isEmpty) { return columnSource; @@ -162,10 +162,21 @@ public class FullDataToRenderDataTransformer } else { - // expand the ColumnArrayView to fit the new larger max vertical size - ColumnArrayView newColumnArrayView = new ColumnArrayView(new LongArrayList(new long[fullDataLength]), fullDataLength, 0, fullDataLength); - setRenderColumnView(level, fullDataMapping, blockX, blockZ, newColumnArrayView, fullDataColumn); - columnArrayView.changeVerticalSizeFrom(newColumnArrayView); + // new LongArrayList(new long[fullDataLength]) + PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(0, 0, 1); + LongArrayList dataArrayList = checkout.getLongArray(0, fullDataLength); + + try + { + // expand the ColumnArrayView to fit the new larger max vertical size + ColumnArrayView newColumnArrayView = new ColumnArrayView(dataArrayList, fullDataLength, 0, fullDataLength); + setRenderColumnView(level, fullDataMapping, blockX, blockZ, newColumnArrayView, fullDataColumn); + columnArrayView.changeVerticalSizeFrom(newColumnArrayView); + } + finally + { + PhantomArrayListPool.INSTANCE.returnCheckout(checkout); + } } } private static void setRenderColumnView( diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 82fabbfc6..8af53a659 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -71,7 +71,7 @@ public class LodDataBuilder int sectionPosZ = getXOrZSectionPosFromChunkPos(chunkWrapper.getChunkPos().getZ()); long pos = DhSectionPos.encode(DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL, sectionPosX, sectionPosZ); - FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos); + FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos); dataSource.isEmpty = false; @@ -140,7 +140,18 @@ public class LodDataBuilder { for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++) { - LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); + LongArrayList longs = dataSource.get( + relBlockX + chunkOffsetX, + relBlockZ + chunkOffsetZ); + if (longs == null) + { + longs = new LongArrayList(chunkWrapper.getHeight() / 4); + } + else + { + longs.clear(); + } + int lastY = chunkWrapper.getExclusiveMaxBuildHeight(); IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ); IBlockStateWrapper blockState = AIR; @@ -388,7 +399,7 @@ public class LodDataBuilder int relSourceBlockX = Math.floorMod(apiChunk.chunkPosX, 4) * LodUtil.CHUNK_WIDTH; int relSourceBlockZ = Math.floorMod(apiChunk.chunkPosZ, 4) * LodUtil.CHUNK_WIDTH; - FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos); + FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos); for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++) { for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index 3dbae5a35..2a20b3a4a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -5,6 +5,7 @@ import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import com.seibel.distanthorizons.core.util.LodUtil; @@ -138,9 +139,8 @@ public abstract class AbstractDataSourceHandler public TDataSource get(long pos) { TDataSource dataSource = null; - try + try(TDTO dto = this.repo.getByKey(pos)) { - TDTO dto = this.repo.getByKey(pos); if (dto != null) { try @@ -152,10 +152,10 @@ public abstract class AbstractDataSourceHandler { // Only log each message type once. // This is done to prevent logging "No compression mode with the value [2]" 10,000 times - // if the user is migrating from a nightly build and used ZStd. + // if the user is migrating from a nightly build and used ZStd. if (CORRUPT_DATA_ERRORS_LOGGED.add(e.getMessage())) { - LOGGER.warn("Corrupted data found at pos [" + DhSectionPos.toString(pos) + "]. Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: " + e.getMessage(), e); + LOGGER.warn("Corrupted data found at pos [" + DhSectionPos.toString(pos) + "]. Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + e.getMessage() + "].", e); } this.repo.deleteWithKey(pos); @@ -252,8 +252,10 @@ public abstract class AbstractDataSourceHandler if (dataModified) { // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(recipientDataSource); - this.repo.save(dto); + try (TDTO dto = this.createDtoFromDataSource(recipientDataSource)) + { + this.repo.save(dto); + } for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java deleted file mode 100644 index cd8da6f4b..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.seibel.distanthorizons.core.file; - -import com.seibel.distanthorizons.core.level.IDhLevel; -import org.jetbrains.annotations.Nullable; - -import java.util.ArrayList; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; - -/** - * Data sources are often very large objects and aren't used for very long. - * This means their frequent construction and garbage collection can result in quite a bit of GC pressure. - * By pooling said data sources and reusing them we can drastically reduce this GC pressure and improve - * performance significantly. - */ -public class DataSourcePool, TDhLevel extends IDhLevel> -{ - /** - * James tested with a static 25 on a 8 core 16 processor machine and didn't have any issues. - * In most cases the number of pooled sources won't probably even get close to the number of processors, - * but just in case the user has a overkill CPU (or config) this should hopefully prevent thrashing. - */ - private static final int MAX_POOLED_SOURCES = Runtime.getRuntime().availableProcessors() * 2; - - private final ArrayList pooledDataSources = new ArrayList<>(); - private final ReentrantLock poolLock = new ReentrantLock(); - - private final Function createEmptyDatasourceFunc; - @Nullable - private final IPrepPooledDataSourceFunc prepDatasourceFunc; - - - - //=============// - // constructor // - //=============// - - public DataSourcePool(Function createEmptyDatasourceFunc, @Nullable IPrepPooledDataSourceFunc prepDatasourceFunc) - { - this.createEmptyDatasourceFunc = createEmptyDatasourceFunc; - this.prepDatasourceFunc = prepDatasourceFunc; - } - - - - //===============// - // pool handlers // - //===============// - - /** - * Returns a cleared data source. - * @see DataSourcePool#getPooledSource(long, boolean) - */ - public TDataSource getPooledSource(long pos) { return this.getPooledSource(pos, true);} - - /** @return an empty data source if non are cached */ - public TDataSource getPooledSource(long pos, boolean clearData) - { - try - { - this.poolLock.lock(); - - int index = this.pooledDataSources.size() - 1; - if (index == -1) - { - // no pooled sources exist - return this.createEmptyDatasourceFunc.apply(pos); - } - else - { - TDataSource dataSource = this.pooledDataSources.remove(index); - - // some data sources may want to handle prep themselves - // (due to needing additional inputs than what this pool keeps track of) - if (this.prepDatasourceFunc != null) - { - this.prepDatasourceFunc.prepDataSource(pos, clearData, dataSource); - } - - return dataSource; - } - } - finally - { - this.poolLock.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link DataSourcePool#getPooledSource} is called. - */ - public void returnPooledDataSource(TDataSource dataSource) - { - if (dataSource == null) - { - return; - } - else if (this.pooledDataSources.size() > MAX_POOLED_SOURCES) - { - return; - } - - try - { - this.poolLock.lock(); - this.pooledDataSources.add(dataSource); - } - finally - { - this.poolLock.unlock(); - } - } - - - - //===============// - // debug methods // - //===============// - - /** Returns how many data sources are in the pool */ - public int size() { return this.pooledDataSources.size(); } - - - - //================// - // helper classes // - //================// - - @FunctionalInterface - public interface IPrepPooledDataSourceFunc, TDhLevel extends IDhLevel> - { - /** @param clearData will be false if the data will be immediately overwritten anyway */ - void prepDataSource(long pos, boolean clearData, TDataSource dataSource); - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java index 456b92052..56682c1c1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java @@ -45,14 +45,18 @@ public class DelayedFullDataSourceSaveCache // update queue // //==============// - public void queueDataSourceForUpdateAndSave(FullDataSourceV2 inputDataSource) + /** + * Writing into memory is done synchronously so inputDataSource can + * be closed after this method finishes. + */ + public void writeDataSourceToMemoryAndQueueSave(FullDataSourceV2 inputDataSource) { long dataSourcePos = inputDataSource.getPos(); this.dataSourceByPosition.compute(dataSourcePos, (inputPos, temporaryDataSource) -> { if (temporaryDataSource == null) { - temporaryDataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(inputPos); + temporaryDataSource = FullDataSourceV2.createEmpty(inputPos); } temporaryDataSource.update(inputDataSource); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index 23df683b7..0039b56e4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -121,9 +121,8 @@ public class FullDataSourceProviderV1 public FullDataSourceV1 get(Long pos) { FullDataSourceV1 dataSource = null; - try + try (FullDataSourceV1DTO dto = this.repo.getByKey(pos)) { - FullDataSourceV1DTO dto = this.repo.getByKey(pos); if (dto != null) { // load from file diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index df0735920..82019ac98 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -21,7 +21,6 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.api.internal.ClientApi; -import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; @@ -41,7 +40,6 @@ import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; -import com.seibel.distanthorizons.core.world.EWorldEnvironment; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; @@ -181,10 +179,11 @@ public class FullDataSourceProviderV2 @Override protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException - { return dto.createPooledDataSource(this.level.getLevelWrapper()); } + { return dto.createDataSource(this.level.getLevelWrapper()); } @Override - protected FullDataSourceV2 makeEmptyDataSource(long pos) { return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true); } + protected FullDataSourceV2 makeEmptyDataSource(long pos) + { return FullDataSourceV2.createEmpty(pos); } @@ -286,7 +285,7 @@ public class FullDataSourceProviderV2 } catch (Exception e) { - LOGGER.error("issue in update for parent pos: " + parentUpdatePos+ " Error: "+e.getMessage(), e); + LOGGER.error("Unexpected in update for parent pos: [" + DhSectionPos.toString(parentUpdatePos) + "] Error: [" + e.getMessage() + "].", e); } finally { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index f57f96b5b..a6590a1ca 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -29,6 +29,8 @@ import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; @@ -36,6 +38,7 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -288,10 +291,10 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im public int getUnsavedDataSourceCount() { return this.delayedFullDataSourceSaveCache.getUnsavedCount(); } - public boolean isFullyGenerated(byte[] columnGenerationSteps) + public boolean isFullyGenerated(ByteArrayList columnGenerationSteps) { - return IntStream.range(0, columnGenerationSteps.length) - .noneMatch(i -> columnGenerationSteps[i] == EDhApiWorldGenerationStep.EMPTY.value); + return IntStream.range(0, columnGenerationSteps.size()) + .noneMatch(i -> columnGenerationSteps.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value); } @Override @@ -307,25 +310,28 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im // don't check any child positions if this position is already fully generated if (this.repo.existsWithKey(pos)) { - byte[] columnGenerationSteps = this.repo.getColumnGenerationStepForPos(pos); - // shouldn't happen, but just in case - if (columnGenerationSteps != null) + try(PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(1, 0, 0)) { - boolean positionFullyGenerated = true; - - // check if any positions are ungenerated - for (int i = 0; i < columnGenerationSteps.length; i++) + ByteArrayList columnGenStepArray = checkout.getByteArray(0); + this.repo.getColumnGenerationStepForPos(pos, columnGenStepArray); + if (!columnGenStepArray.isEmpty()) { - if (columnGenerationSteps[i] == EDhApiWorldGenerationStep.EMPTY.value) + boolean positionFullyGenerated = true; + + // check if any positions are ungenerated + for (int i = 0; i < columnGenStepArray.size(); i++) { - positionFullyGenerated = false; - break; + if (columnGenStepArray.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value) + { + positionFullyGenerated = false; + break; + } + } + + if (positionFullyGenerated) + { + return new LongArrayList(); } - } - - if (positionFullyGenerated) - { - return new LongArrayList(); } } } @@ -349,36 +355,42 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im } else { - byte[] columnGenerationSteps = this.repo.getColumnGenerationStepForPos(genPos); - if (columnGenerationSteps == null) - { - // shouldn't happen, but just in case - return; - } - EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT; - checkWorldGenLoop: - for (int x = 0; x < FullDataSourceV2.WIDTH; x++) + try(PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(1, 0, 0)) { - for (int z = 0; z < FullDataSourceV2.WIDTH; z++) + ByteArrayList columnGenerationSteps = checkout.getByteArray(0); + this.repo.getColumnGenerationStepForPos(genPos, columnGenerationSteps); + if (columnGenerationSteps.isEmpty()) { - int index = FullDataSourceV2.relativePosToIndex(x, z); - byte genStepValue = columnGenerationSteps[index]; - - if (genStepValue < currentMinWorldGenStep.value) + // shouldn't happen, but just in case + return; + } + + + + checkWorldGenLoop: + for (int x = 0; x < FullDataSourceV2.WIDTH; x++) + { + for (int z = 0; z < FullDataSourceV2.WIDTH; z++) { - EDhApiWorldGenerationStep newWorldGenStep = EDhApiWorldGenerationStep.fromValue(genStepValue); - if (newWorldGenStep != null && newWorldGenStep.value < currentMinWorldGenStep.value) + int index = FullDataSourceV2.relativePosToIndex(x, z); + byte genStepValue = columnGenerationSteps.getByte(index); + + if (genStepValue < currentMinWorldGenStep.value) { - currentMinWorldGenStep = newWorldGenStep; + EDhApiWorldGenerationStep newWorldGenStep = EDhApiWorldGenerationStep.fromValue(genStepValue); + if (newWorldGenStep != null && newWorldGenStep.value < currentMinWorldGenStep.value) + { + currentMinWorldGenStep = newWorldGenStep; + } + } + + if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY) + { + // queue the task + break checkWorldGenLoop; } - } - - if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY) - { - // queue the task - break checkWorldGenLoop; } } } @@ -447,7 +459,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im { return (dataSource) -> { - GeneratedFullDataSourceProvider.this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(dataSource); + GeneratedFullDataSourceProvider.this.delayedFullDataSourceSaveCache.writeDataSourceToMemoryAndQueueSave(dataSource); }; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java index 82b08c54b..a6320480d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java @@ -25,7 +25,9 @@ import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.generation.RemoteWorldRetrievalQueue; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.level.WorldGenModule; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.multiplayer.client.SyncOnLoadRequestQueue; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.io.File; @@ -39,6 +41,8 @@ import java.util.concurrent.TimeUnit; */ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvider { + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + @Nullable private final SyncOnLoadRequestQueue syncOnLoadRequestQueue; private final Set visitedPositions = Collections.newSetFromMap(CacheBuilder.newBuilder() @@ -95,7 +99,7 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide Long timestamp = this.getTimestampForPos(pos); if (timestamp != null) { - this.syncOnLoadRequestQueue.submitRequest(pos, timestamp, this.delayedFullDataSourceSaveCache::queueDataSourceForUpdateAndSave); + this.syncOnLoadRequestQueue.submitRequest(pos, timestamp, this.delayedFullDataSourceSaveCache::writeDataSourceToMemoryAndQueueSave); } return super.get(pos); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 13f89c920..f0c9811f2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -392,15 +392,22 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb try { IChunkWrapper chunk = WRAPPER_FACTORY.createChunkWrapper(generatedObjectArray); - FullDataSourceV2 dataSource = LodDataBuilder.createFromChunk(chunk); - LodUtil.assertTrue(dataSource != null); - dataSourceConsumer.accept(dataSource); + try (FullDataSourceV2 dataSource = LodDataBuilder.createFromChunk(chunk)) + { + LodUtil.assertTrue(dataSource != null); + dataSourceConsumer.accept(dataSource); + } } catch (ClassCastException e) { LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e); Config.Common.WorldGenerator.enableDistantGeneration.set(false); } + catch (Exception e) + { + LOGGER.error("Unexpected world generator error. Error: [" + e.getMessage() + "]. World generator disabled.", e); + Config.Common.WorldGenerator.enableDistantGeneration.set(false); + } } ); } @@ -435,7 +442,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb case API_DATA_SOURCES: { // done to reduce GC overhead - FullDataSourceV2 pooledDataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(requestPos); + FullDataSourceV2 pooledDataSource = FullDataSourceV2.createEmpty(requestPos); // set here so the API user doesn't have to pass in this value anywhere themselves pooledDataSource.setRunApiChunkValidation(this.generator.runApiValidation()); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java index 78356d5fe..1382fd9a8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java @@ -27,7 +27,6 @@ import com.seibel.distanthorizons.core.generation.DhLightingEngine; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler; import com.seibel.distanthorizons.core.render.renderer.generic.CloudRenderHandler; import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer; import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO; @@ -147,27 +146,30 @@ public abstract class AbstractDhLevel implements IDhLevel @Override public void updateChunkAsync(IChunkWrapper chunkWrapper, int chunkHash) { - FullDataSourceV2 dataSource = FullDataSourceV2.createFromChunk(chunkWrapper); - if (dataSource == null) + // data source synchronously written to memory so it can be safely closed + try (FullDataSourceV2 dataSource = FullDataSourceV2.createFromChunk(chunkWrapper)) { - // This can happen if, among other reasons, a chunk save is superseded by a later event - return; - } - - - this.updatedChunkPosSetBySectionPos.compute(dataSource.getPos(), (dataSourcePos, chunkPosSet) -> - { - if (chunkPosSet == null) + if (dataSource == null) { - chunkPosSet = new HashSet<>(); + // This can happen if, among other reasons, a chunk save is superseded by a later event + return; } - chunkPosSet.add(chunkWrapper.getChunkPos()); - return chunkPosSet; - }); - this.updatedChunkHashesByChunkPos.put(chunkWrapper.getChunkPos(), chunkHash); - - // batch updates to reduce overhead when flying around or breaking/placing a lot of blocks in an area - this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(dataSource); + + + this.updatedChunkPosSetBySectionPos.compute(dataSource.getPos(), (dataSourcePos, chunkPosSet) -> + { + if (chunkPosSet == null) + { + chunkPosSet = new HashSet<>(); + } + chunkPosSet.add(chunkWrapper.getChunkPos()); + return chunkPosSet; + }); + this.updatedChunkHashesByChunkPos.put(chunkWrapper.getChunkPos(), chunkHash); + + // batch updates to reduce overhead when flying around or breaking/placing a lot of blocks in an area + this.delayedFullDataSourceSaveCache.writeDataSourceToMemoryAndQueueSave(dataSource); + } } private void onDataSourceSave(FullDataSourceV2 fullDataSource) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 4d67fa2dc..3bb457ad3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -145,7 +145,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel } this.beaconBeamDataHandler.setBeaconBeamsForPos(dataSourceDto.pos, message.payload.beaconBeams); - this.updateDataSourcesAsync(dataSourceDto.createPooledDataSource(this.levelWrapper)); + this.updateDataSourcesAsync(dataSourceDto.createDataSource(this.levelWrapper)); } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java b/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java index 2636d8bd4..c4a5b3931 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java @@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.jar.ModJarInfo; import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.RenderBufferHandler; import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer; @@ -30,7 +31,6 @@ import com.seibel.distanthorizons.core.util.threading.RateLimitedThreadPoolExecu import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.world.AbstractDhWorld; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; -import com.seibel.distanthorizons.coreapi.DependencyInjection.DependencyInjector; import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.coreapi.util.StringUtil; import org.apache.logging.log4j.LogManager; @@ -107,6 +107,9 @@ public class F3Screen messageList.add(getThreadPoolStatString("LOD Builder", lodBuilderPool)); messageList.add(getThreadPoolStatString("Buffer Builder", bufferBuilderPool)); messageList.add(""); + // object pools + PhantomArrayListPool.INSTANCE.addDebugMenuStringsToList(messageList); + messageList.add(""); // chunk updates messageList.add(SharedApi.INSTANCE.getDebugMenuString()); messageList.add(""); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java index df6dac86a..0af386bf0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java @@ -20,7 +20,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.ratelimiting.SupplierBasedRateLimiter; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.world.DhApiWorldProxy; @@ -30,7 +29,6 @@ import org.apache.logging.log4j.LogManager; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import java.awt.*; -import java.io.IOException; import java.util.*; import java.util.List; import java.util.concurrent.*; @@ -235,11 +233,12 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende try { this.level.getBeaconBeamDataHandler().setBeaconBeamsForPos(dataSourceDto.pos, response.payload.beaconBeams); - FullDataSourceV2 fullDataSource = dataSourceDto.createPooledDataSource(this.level.getLevelWrapper()); - entry.dataSourceConsumer.accept(fullDataSource); - FullDataSourceV2.DATA_SOURCE_POOL.returnPooledDataSource(fullDataSource); + try (FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper())) + { + entry.dataSourceConsumer.accept(fullDataSource); + } } - catch (IOException | DataCorruptedException | InterruptedException e) + catch (Exception e) { throw new RuntimeException(e); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayload.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayload.java index c550b385c..d4cad0227 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayload.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayload.java @@ -45,10 +45,11 @@ public class FullDataPayload implements INetworkObject, AutoCloseable try { EDhApiDataCompressionMode compressionMode = Config.Common.LodBuilding.dataCompression.get(); - FullDataSourceV2DTO dataSourceDto = FullDataSourceV2DTO.CreateFromDataSource(fullDataSource, compressionMode); - - this.dtoBuffer = ByteBufAllocator.DEFAULT.buffer(); - dataSourceDto.encode(this.dtoBuffer); + try (FullDataSourceV2DTO dataSourceDto = FullDataSourceV2DTO.CreateFromDataSource(fullDataSource, compressionMode)) + { + this.dtoBuffer = ByteBufAllocator.DEFAULT.buffer(); + dataSourceDto.encode(this.dtoBuffer); + } } catch (IOException e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayloadReceiver.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayloadReceiver.java index 6fda2b15c..f9af0b20e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayloadReceiver.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayloadReceiver.java @@ -76,7 +76,7 @@ public class FullDataPayloadReceiver implements AutoCloseable try { - return INetworkObject.decodeToInstance(FullDataSourceV2DTO.CreateEmptyDataSource(), compositeByteBuffer); + return INetworkObject.decodeToInstance(FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding(), compositeByteBuffer); } finally { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java new file mode 100644 index 000000000..1618b0fef --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java @@ -0,0 +1,103 @@ +package com.seibel.distanthorizons.core.pooling; + +import com.seibel.distanthorizons.core.util.ListUtil; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.shorts.ShortArrayList; + +import java.util.ArrayList; + +/** + * This keeps track of all the poolable + * arrays that can be retrieved via the {@link PhantomArrayListPool}. + * + * @see PhantomArrayListParent + * @see PhantomArrayListPool + */ +public class PhantomArrayListCheckout implements AutoCloseable +{ + private final ArrayList byteArrayLists = new ArrayList<>(); + private final ArrayList shortArrayLists = new ArrayList<>(); + private final ArrayList longArrayLists = new ArrayList<>(); + + + + //=========// + // setters // + //=========// + + public void addByteArrayList(ByteArrayList list) { this.byteArrayLists.add(list); } + public void addShortArrayList(ShortArrayList list) { this.shortArrayLists.add(list); } + public void addLongArrayList(LongArrayList list) { this.longArrayLists.add(list); } + + + + //=========// + // getters // + //=========// + + public int getByteArrayCount() { return this.byteArrayLists.size(); } + public int getShortArrayCount() { return this.shortArrayLists.size(); } + public int getLongArrayCount() { return this.longArrayLists.size(); } + + + + public ByteArrayList getByteArray(int index) { return this.getByteArray(index, 0); } + public ByteArrayList getByteArray(int index, int size) + { + ByteArrayList list = this.byteArrayLists.get(index); + if (size != 0) + { + ListUtil.clearAndSetSize(list, size); + } + else + { + list.clear(); + } + return list; + } + + public ShortArrayList getShortArray(int index) { return this.getShortArray(index, 0); } + public ShortArrayList getShortArray(int index, int size) + { + ShortArrayList list = this.shortArrayLists.get(index); + if (size != 0) + { + ListUtil.clearAndSetSize(list, size); + } + else + { + list.clear(); + } + return list; + } + + public LongArrayList getLongArray(int index) { return this.getLongArray(index, 0); } + public LongArrayList getLongArray(int index, int size) + { + LongArrayList list = this.longArrayLists.get(index); + if (size != 0) + { + ListUtil.clearAndSetSize(list, size); + } + else + { + list.clear(); + } + return list; + } + + + public ArrayList getAllByteArrays() { return this.byteArrayLists; } + public ArrayList getAllShortArrays() { return this.shortArrayLists; } + public ArrayList getAllLongArrays() { return this.longArrayLists; } + + + + @Override + public void close() + { + PhantomArrayListPool.INSTANCE.returnCheckout(this); + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java new file mode 100644 index 000000000..33b93bffa --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java @@ -0,0 +1,148 @@ +package com.seibel.distanthorizons.core.pooling; + +import com.seibel.distanthorizons.core.util.ThreadUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.ref.PhantomReference; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Any object that needs pooled arrays should extend this object. + * This handles setting up and tracking the necessary {@link PhantomReference}'s + * needed to make sure none of the arrays are leaked. + * However, if possible, the implementing object should be closed + * instead via a try-resource block as that will reduce the number of + * unnecessary arrays created. + * + * @see PhantomArrayListCheckout + * @see PhantomArrayListPool + */ +public abstract class PhantomArrayListParent implements AutoCloseable +{ + private static final Logger LOGGER = LogManager.getLogger(); + + private static final ConcurrentHashMap, PhantomArrayListCheckout> + PHANTOM_REF_TO_CHECKOUT = new ConcurrentHashMap<>(); + private static final ReferenceQueue PHANTOM_REF_QUEUE = new ReferenceQueue<>(); + + private static final int PHANTOM_REF_CHECK_TIME_IN_MS = 5 * 1000; + private static final ThreadPoolExecutor RECYCLER_THREAD = ThreadUtil.makeSingleThreadPool("Phantom Array Recycler"); + + + + private final PhantomReference phantomReference; + + /** + * It's recommended to set this as null after the child's constructor + * finishes to show the pooled arrays have all been accessed + */ + protected PhantomArrayListCheckout pooledArraysCheckout; + + + + //====================// + // static constructor // + //====================// + + static { RECYCLER_THREAD.execute(() -> runPhantomReferenceCleanupLoop()); } + + private static void runPhantomReferenceCleanupLoop() + { + while (true) + { + try + { + try + { + Thread.sleep(PHANTOM_REF_CHECK_TIME_IN_MS); + } + catch (InterruptedException ignore) { } + + + int returnedByteArrayCount = 0; + int returnedShortArrayCount = 0; + int returnedLongArrayCount = 0; + Reference phantomRef = PHANTOM_REF_QUEUE.poll(); + while (phantomRef != null) + { + // return the pooled arrays + PhantomArrayListCheckout checkout = PHANTOM_REF_TO_CHECKOUT.remove(phantomRef); + if (checkout != null) + { + returnedByteArrayCount += checkout.getByteArrayCount(); + returnedShortArrayCount += checkout.getShortArrayCount(); + returnedLongArrayCount += checkout.getLongArrayCount(); + PhantomArrayListPool.INSTANCE.returnCheckout(checkout); + } + else + { + // shouldn't happen, but just in case + LOGGER.warn("Unable to find checkout for phantom reference ["+phantomRef+"], arrays will need to be recreated."); + } + + phantomRef = PHANTOM_REF_QUEUE.poll(); + } + + if (returnedByteArrayCount != 0 && returnedLongArrayCount != 0) + { + // we only want to log when arrays have been returned + //LOGGER.info("Returned byte:["+F3Screen.NUMBER_FORMAT.format(returnedByteArrayCount)+"], short:["+F3Screen.NUMBER_FORMAT.format(returnedShortArrayCount)+"], long:["+F3Screen.NUMBER_FORMAT.format(returnedLongArrayCount)+"]."); + + // since this is just for debugging it only needs to be recalculated once in a while + PhantomArrayListPool.INSTANCE.recalculateSizeForDebugging(); + } + } + catch (Exception e) + { + LOGGER.error("Unexpected error in phantom pool return thread, error: [" + e.getMessage() + "].", e); + } + } + } + + + + //=============// + // constructor // + //=============// + + /** The Array counts can be 0 or greater. */ + public PhantomArrayListParent(int byteArrayCount, int shortArrayCount, int longArrayCount) + { + if (byteArrayCount < 0 || shortArrayCount < 0 || longArrayCount < 0) + { + throw new IllegalArgumentException("Can't get a negative number of pooled arrays."); + } + + this.phantomReference = new PhantomReference<>(this, PHANTOM_REF_QUEUE); + this.pooledArraysCheckout = PhantomArrayListPool.INSTANCE.checkoutArrays(byteArrayCount, shortArrayCount, longArrayCount); + PHANTOM_REF_TO_CHECKOUT.put(this.phantomReference, this.pooledArraysCheckout); + } + + + + //================// + // base overrides // + //================// + + @Override + public void close() //throws Exception + { + try + { + this.phantomReference.clear(); + PhantomArrayListCheckout checkout = PHANTOM_REF_TO_CHECKOUT.remove(this.phantomReference); + PhantomArrayListPool.INSTANCE.returnCheckout(checkout); + } + catch (Exception e) + { + LOGGER.error("", e); + } + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java new file mode 100644 index 000000000..e0cad1671 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java @@ -0,0 +1,269 @@ +package com.seibel.distanthorizons.core.pooling; + +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.logging.f3.F3Screen; +import com.seibel.distanthorizons.coreapi.util.StringUtil; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.shorts.ShortArrayList; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Supplier; + +/** + * DH uses a lot of potentially large arrays of {@link Byte}s and {@link Long}s. + * In order to reduce Garbage Collector (GC) stuttering and array allocation overhead + * we pool these arrays when possible.

+ * + * How pooled arrays can be returned:
+ * 1. Closing the {@link PhantomArrayListParent}
+ * The fastest and most efficient method of returning pooled arrays + * is to call {@link AutoCloseable#close()}.

+ * + * 2. {@link PhantomArrayListParent} Garbage Collection
+ * Some objects are used across many different threads and + * cleanly closing them is impossible, so when the {@link PhantomArrayListParent} + * is automatically garbage collected we recover and recycle any + * arrays it checked out. + * This is less efficient since it may allow a lot of additional arrays to + * be created while we wait for the garbage collector to run, but + * does prevent any leaks from {@link PhantomArrayListParent} that weren't closed. + */ +public class PhantomArrayListPool +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + public static final PhantomArrayListPool INSTANCE = new PhantomArrayListPool(); + + + /** needed since our pools aren't thread safe */ + private final ReentrantLock poolLock = new ReentrantLock(); + + private final ArrayList pooledByteArrays = new ArrayList<>(); + private final ArrayList pooledShortArrays = new ArrayList<>(); + private final ArrayList pooledLongArrays = new ArrayList<>(); + + /** counts how many byte arrays have been created by this pool */ + private final AtomicInteger totalByteArrayCountRef = new AtomicInteger(0); + /** counts how many short arrays have been created by this pool */ + private final AtomicInteger totalShortArrayCountRef = new AtomicInteger(0); + /** counts how many long arrays have been created by this pool */ + private final AtomicInteger totalLongArrayCountRef = new AtomicInteger(0); + + /** used for debugging, represents an estimate for how many bytes the byte[] pool contains */ + private long lastBytePoolSizeInBytes = -1; + /** used for debugging, represents an estimate for how many bytes the short[] pool contains */ + private long lastShortPoolSizeInBytes = -1; + /** used for debugging, represents an estimate for how many bytes the long[] pool contains */ + private long lastLongPoolSizeInBytes = -1; + + + + //=============// + // constructor // + //=============// + + private PhantomArrayListPool() + { + // create initial arrays + PhantomArrayListCheckout checkout = this.checkoutArrays(2_000, 100, 100_000); + this.returnCheckout(checkout); + } + + + + //==============// + // get checkout // + //==============// + + public PhantomArrayListCheckout checkoutArrays(int byteArrayCount, int shortArrayCount, int longArrayCount) + { + try + { + this.poolLock.lock(); + + PhantomArrayListCheckout checkout = new PhantomArrayListCheckout(); + + // byte + for (int i = 0; i < byteArrayCount; i++) + { + checkout.addByteArrayList(getPooledArray(this.pooledByteArrays, this::createEmptyByteArrayList)); + } + + // short + for (int i = 0; i < shortArrayCount; i++) + { + checkout.addShortArrayList(getPooledArray(this.pooledShortArrays, this::createEmptyShortArrayList)); + } + + // long + for (int i = 0; i < longArrayCount; i++) + { + checkout.addLongArrayList(getPooledArray(this.pooledLongArrays, this::createEmptyLongArrayList)); + } + + return checkout; + } + finally + { + this.poolLock.unlock(); + } + } + + + // array constructors // + + private ByteArrayList createEmptyByteArrayList() + { + //LOGGER.error("created new byte array"); + this.totalByteArrayCountRef.getAndIncrement(); + return new ByteArrayList(); + } + private ShortArrayList createEmptyShortArrayList() + { + //LOGGER.error("created new short array"); + this.totalShortArrayCountRef.getAndIncrement(); + return new ShortArrayList(); + } + private LongArrayList createEmptyLongArrayList() + { + //LOGGER.error("created new long array"); + this.totalLongArrayCountRef.getAndIncrement(); + return new LongArrayList(); + } + + + // internal pool handler // + + private static > T getPooledArray(ArrayList pool, Supplier emptyArrayCreatorFunc) + { + int index = pool.size() - 1; + if (index == -1) + { + // no pooled sources exist + return emptyArrayCreatorFunc.get(); + } + else + { + T array = pool.remove(index); + array.clear(); + return array; + } + } + + + + //=================// + // return checkout // + //=================// + + public void returnCheckout(PhantomArrayListCheckout checkout) + { + if (checkout == null) + { + throw new IllegalArgumentException("Null phantom checkout, memory leak in progress..."); + } + + this.poolLock.lock(); + + try + { + // In James' testing pooling the checkout object wasn't necessary + // since it is relatively small and short lived it appears + // the GC can handle quickly discarding it. + + this.pooledByteArrays.addAll(checkout.getAllByteArrays()); + this.pooledShortArrays.addAll(checkout.getAllShortArrays()); + this.pooledLongArrays.addAll(checkout.getAllLongArrays()); + + //LOGGER.info("Returned ["+checkout.byteArrayLists.size()+"/"+this.pooledByteArrays.size()+"] bytes and ["+checkout.longArrayLists.size()+"/"+this.pooledLongArrays.size()+"] longs.");\ + } + finally + { + this.poolLock.unlock(); + } + } + + + + //===============// + // debug methods // + //===============// + + public void addDebugMenuStringsToList(List messageList) + { + // total (all time created) count + String byteArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalByteArrayCountRef.get()); + String shortArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalShortArrayCountRef.get()); + String longArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalLongArrayCountRef.get()); + + // inactive items in pool + String bytePoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledByteArrays.size()); + String shortPoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledShortArrays.size()); + String longPoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledLongArrays.size()); + + // pool byte size + String bytePoolSizeInBytes = (this.lastBytePoolSizeInBytes != -1) + ? " ~" + StringUtil.convertBytesToHumanReadable(this.lastBytePoolSizeInBytes) + : ""; + String shortPoolSizeInBytes = (this.lastShortPoolSizeInBytes != -1) + ? " ~" + StringUtil.convertBytesToHumanReadable(this.lastShortPoolSizeInBytes) + : ""; + String longPoolSizeInBytes = (this.lastLongPoolSizeInBytes != -1) + ? " ~" + StringUtil.convertBytesToHumanReadable(this.lastLongPoolSizeInBytes) + : ""; + + messageList.add("Pools:"); + messageList.add("byte[]: "+bytePoolCount+"/"+byteArrayTotalCount + bytePoolSizeInBytes); + messageList.add("short[]: "+shortPoolCount+"/"+shortArrayTotalCount + shortPoolSizeInBytes); + messageList.add("long[]: "+longPoolCount+"/"+longArrayTotalCount + longPoolSizeInBytes); + } + + /** + * shouldn't be called on the render thread as it can + * take 10's of milliseconds to complete. + */ + public void recalculateSizeForDebugging() + { + this.poolLock.lock(); + try + { + // byte + long bytePoolByteSize = estimateMemoryUsage(this.pooledByteArrays, Byte.BYTES); + this.lastBytePoolSizeInBytes = Math.max(bytePoolByteSize, this.lastBytePoolSizeInBytes); + + // short + long shortPoolByteSize = estimateMemoryUsage(this.pooledShortArrays, Short.BYTES); + this.lastShortPoolSizeInBytes = Math.max(shortPoolByteSize, this.lastShortPoolSizeInBytes); + + // long + long longPoolByteSize = estimateMemoryUsage(this.pooledLongArrays, Long.BYTES); + this.lastLongPoolSizeInBytes = Math.max(longPoolByteSize, this.lastLongPoolSizeInBytes); + } + finally + { + this.poolLock.unlock(); + } + } + + private static > long estimateMemoryUsage(ArrayList pool, long elementSizeInBytes) + { + long longByteSize = 0; + for (int i = 0; i < pool.size(); i++) + { + // Object overhead + capacity of underlying array * size of Long (8 bytes) + long overhead = Byte.SIZE * 4; + long arraySize = (long)pool.get(i).size() * elementSizeInBytes; + longByteSize += overhead + arraySize; + } + return longByteSize; + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index 64bf74423..48f36c70d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -434,6 +434,7 @@ public class DhSectionPos // Base methods // //==============// + /** Example: "6*1,-3" */ public static String toString(long pos) { return getDetailLevel(pos) + "*" + getX(pos) + "," + getZ(pos); } public static int hashCode(long pos) { return Long.hashCode(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 40aaa75b1..771e7ea75 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -27,7 +27,6 @@ import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodQuad import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; -import com.seibel.distanthorizons.core.file.DataSourcePool; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -632,7 +631,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable /** * Used to keep track of whether a {@link ColumnRenderSource} {@link CompletableFuture} - * is in use or not, and if not in use returns the data source to the {@link DataSourcePool}.

+ * is in use or not.

* * This reduces GC overhead by pooling shared {@link ColumnRenderSource}. */ @@ -687,7 +686,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable ColumnRenderSource source = this.future.getNow(null); if (source != null) { - ColumnRenderSource.DATA_SOURCE_POOL.returnPooledDataSource(source); + source.close(); } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java index b3696d065..265d3e212 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -26,9 +26,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; import java.io.InputStream; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.ArrayList; import java.util.Map; import java.util.Scanner; @@ -130,17 +128,23 @@ public class DatabaseUpdater } catch (SQLException e) { - connection.rollback(); - LOGGER.error( - "Unexpected SQL Error: ["+e.getMessage()+"] " + - "returned for auto update script: [" + resource.name + "], " + - "query: [" + fileUpdateSqlArray[sqlIndex] + "]. " + - "Changes should have been rolled back.", new SQLException()); + LOGGER.error("Unexpected SQL Error: ["+e.getMessage()+"] \n" + + "returned for auto update script: [" + resource.name + "], \n" + + "query: [" + fileUpdateSqlArray[sqlIndex] + "]. \n" + + "Changes should have been rolled back.", + new SQLException() + ); + + if (transactScript) + { + connection.rollback(); + } throw e; } if (transactScript) - { + { + // revert to default setting connection.setAutoCommit(true); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/BeaconBeamDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/BeaconBeamDTO.java index 0671a78dd..c59d967bb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/BeaconBeamDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/BeaconBeamDTO.java @@ -46,15 +46,6 @@ public class BeaconBeamDTO implements IBaseDTO, INetworkObject - //===========// - // overrides // - //===========// - - @Override - public DhBlockPos getKey() { return this.blockPos; } - - - //=========// // network // //=========// @@ -73,4 +64,18 @@ public class BeaconBeamDTO implements IBaseDTO, INetworkObject this.color = new Color(in.readInt()); } + + //===========// + // overrides // + //===========// + + @Override + public DhBlockPos getKey() { return this.blockPos; } + + @Override + public void close() + { /* no closing needed */ } + + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/ChunkHashDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/ChunkHashDTO.java index 8e532057e..f6bc9c75e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/ChunkHashDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/ChunkHashDTO.java @@ -26,6 +26,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; @@ -69,4 +70,10 @@ public class ChunkHashDTO implements IBaseDTO @Override public DhChunkPos getKey() { return this.pos; } + @Override + public void close() + { /* no closing needed */ } + + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java index 9b849967f..a228e6983 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java @@ -85,5 +85,8 @@ public class FullDataSourceV1DTO implements IBaseDTO @Override public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); } + @Override + public void close() + { /* no closing needed */ } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 2993359b4..7c1ca7831 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -23,27 +23,29 @@ import com.google.common.base.MoreObjects; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.api.internal.ClientApi; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.network.INetworkObject; import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.ListUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import io.netty.buffer.ByteBuf; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.jetbrains.annotations.NotNull; import java.io.*; -import java.util.zip.Adler32; -import java.util.zip.CheckedOutputStream; /** handles storing {@link FullDataSourceV2}'s in the database. */ -public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject +public class FullDataSourceV2DTO + extends PhantomArrayListParent + implements IBaseDTO, INetworkObject, AutoCloseable { public static final boolean VALIDATE_INPUT_DATAPOINTS = true; @@ -55,18 +57,14 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject /** only for the data array */ public int dataChecksum; - // TODO pool byte arrays, and use JDBC blobs or stream said arrays, to potentially reduce GC load - //resultSet.getBinaryStream(0); - //resultSet.getBlob(0).free(); - - public byte[] compressedDataByteArray; + public ByteArrayList compressedDataByteArray; /** @see EDhApiWorldGenerationStep */ - public byte[] compressedColumnGenStepByteArray; + public ByteArrayList compressedColumnGenStepByteArray; /** @see EDhApiWorldCompressionMode */ - public byte[] compressedWorldCompressionModeByteArray; + public ByteArrayList compressedWorldCompressionModeByteArray; - public byte[] compressedMappingByteArray; + public ByteArrayList compressedMappingByteArray; public byte dataFormatVersion; public byte compressionModeValue; @@ -78,56 +76,47 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject - //=============// - // constructor // - //=============// + //==============// + // constructors // + //==============// public static FullDataSourceV2DTO CreateFromDataSource(FullDataSourceV2 dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException { - byte[] dataPointByteArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum); - byte[] compressedWorldGenStepByteArray = writeGenerationStepsToBlob(dataSource.columnGenerationSteps, compressionModeEnum); - byte[] compressedWorldCompressionModeByteArray = writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, compressionModeEnum); - byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum); + FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding(); - int checksum = (dataSource.mapping.hashCode() * 4217) + dataSource.hashCode(); + // populate arrays + writeDataSourceDataArrayToBlob(dataSource.dataPoints, dto.compressedDataByteArray, compressionModeEnum); + writeGenerationStepsToBlob(dataSource.columnGenerationSteps, dto.compressedColumnGenStepByteArray, compressionModeEnum); + writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, dto.compressedWorldCompressionModeByteArray, compressionModeEnum); + writeDataMappingToBlob(dataSource.mapping, dto.compressedMappingByteArray, compressionModeEnum); - return new FullDataSourceV2DTO( - dataSource.getPos(), - checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum.value, dataPointByteArray, - dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, - mappingByteArray, dataSource.applyToParent, - dataSource.levelMinY - ); + // populate individual variables + { + dto.pos = dataSource.getPos(); + dto.dataChecksum = (dataSource.mapping.hashCode() * 4217) + dataSource.hashCode(); + dto.dataFormatVersion = FullDataSourceV2.DATA_FORMAT_VERSION; + dto.compressionModeValue = compressionModeEnum.value; + dto.lastModifiedUnixDateTime = dataSource.lastModifiedUnixDateTime; + dto.createdUnixDateTime = dataSource.createdUnixDateTime; + dto.applyToParent = dataSource.applyToParent; + dto.levelMinY = dataSource.levelMinY; + } + + return dto; } /** Should only be used for subsequent decoding */ - public static FullDataSourceV2DTO CreateEmptyDataSource() { return new FullDataSourceV2DTO(); } - private FullDataSourceV2DTO() { } - - public FullDataSourceV2DTO( - long pos, - int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, byte compressionModeValue, byte[] compressedDataByteArray, - long lastModifiedUnixDateTime, long createdUnixDateTime, - byte[] compressedMappingByteArray, boolean applyToParent, - int levelMinY) + public static FullDataSourceV2DTO CreateEmptyDataSourceForDecoding() { return new FullDataSourceV2DTO(); } + private FullDataSourceV2DTO() { - this.pos = pos; - this.dataChecksum = dataChecksum; - this.compressedColumnGenStepByteArray = compressedColumnGenStepByteArray; - this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray; + super(4, 0, 0); - this.dataFormatVersion = dataFormatVersion; - this.compressionModeValue = compressionModeValue; + this.compressedDataByteArray = this.pooledArraysCheckout.getByteArray(0); + this.compressedColumnGenStepByteArray = this.pooledArraysCheckout.getByteArray(1); + this.compressedWorldCompressionModeByteArray = this.pooledArraysCheckout.getByteArray(2); + this.compressedMappingByteArray = this.pooledArraysCheckout.getByteArray(3); - this.compressedDataByteArray = compressedDataByteArray; - this.compressedMappingByteArray = compressedMappingByteArray; - - this.applyToParent = applyToParent; - - this.lastModifiedUnixDateTime = lastModifiedUnixDateTime; - this.createdUnixDateTime = createdUnixDateTime; - - this.levelMinY = levelMinY; + this.pooledArraysCheckout = null; } @@ -136,15 +125,13 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject // data source population // //========================// - public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException + public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { - FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(this.pos, false); - return this.populateDataSource(dataSource, levelWrapper); + FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos); + this.internalPopulateDataSource(dataSource, levelWrapper, false); + return dataSource; } - public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException - { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } - /** * May be missing one or more data fields.
* Designed to be used without access to Minecraft or any supporting objects. @@ -173,9 +160,9 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject } - dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, compressionModeEnum); - dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, compressionModeEnum); - dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, compressionModeEnum); + readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); + readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum); + readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test @@ -211,7 +198,7 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject // (de)serializing // //=================// - private static byte[] writeDataSourceDataArrayToBlob(LongArrayList[] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static void writeDataSourceDataArrayToBlob(LongArrayList[] inputDataArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -225,7 +212,7 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; for (int xz = 0; xz < dataArrayLength; xz++) { - LongArrayList dataColumn = dataArray[xz]; + LongArrayList dataColumn = inputDataArray[xz]; // write column length short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; @@ -244,19 +231,17 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject // generate the checksum compressedOut.flush(); byteArrayOutputStream.close(); - - return byteArrayOutputStream.toByteArray(); + outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); } - private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException + private static void readBlobToDataSourceDataArray(ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.toByteArray()); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); // read the data int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; - LongArrayList[] dataArray = new LongArrayList[dataArrayLength]; - for (int xz = 0; xz < dataArray.length; xz++) + for (int xz = 0; xz < dataArrayLength; xz++) { // read the column length short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later @@ -265,7 +250,8 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject throw new DataCorruptedException("Read DataSource Blob data at index ["+xz+"], column length ["+dataColumnLength+"] should be greater than zero."); } - LongArrayList dataColumn = new LongArrayList(new long[dataColumnLength]); + LongArrayList dataColumn = outputDataLongArray[xz]; + ListUtil.clearAndSetSize(dataColumn, dataColumnLength); // read column data (will be skipped if no data was present) for (int y = 0; y < dataColumnLength; y++) @@ -277,38 +263,32 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject } dataColumn.set(y, dataPoint); } - - dataArray[xz] = dataColumn; } - - - return dataArray; } - private static byte[] writeGenerationStepsToBlob(byte[] columnGenStepByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static void writeGenerationStepsToBlob(ByteArrayList inputColumnGenStepByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum); - compressedOut.write(columnGenStepByteArray); + for (int i = 0; i < inputColumnGenStepByteArray.size(); i++) + { + compressedOut.writeByte(inputColumnGenStepByteArray.getByte(i)); + } compressedOut.flush(); byteArrayOutputStream.close(); - - return byteArrayOutputStream.toByteArray(); + outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); } - private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException + private static void readBlobToGenerationSteps(ByteArrayList inputCompressedDataByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.toByteArray()); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); try { - byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; - compressedIn.readFully(columnGenStepByteArray); - - return columnGenStepByteArray; + compressedIn.readFully(outputByteArray.elements(), 0, FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH); } catch (EOFException e) { @@ -317,31 +297,37 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject } - private static byte[] writeWorldCompressionModeToBlob(byte[] worldCompressionModeByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static void writeWorldCompressionModeToBlob(ByteArrayList inputWorldCompressionModeByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum); - compressedOut.write(worldCompressionModeByteArray); + for (int i = 0; i < inputWorldCompressionModeByteArray.size(); i++) + { + compressedOut.write(inputWorldCompressionModeByteArray.getByte(i)); + } compressedOut.flush(); byteArrayOutputStream.close(); - - return byteArrayOutputStream.toByteArray(); + outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); } - private static byte[] readBlobToWorldCompressionMode(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + private static void readBlobToWorldCompressionMode(ByteArrayList inputCompressedDataByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.toByteArray()); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); - byte[] worldCompressionModeByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; - compressedIn.readFully(worldCompressionModeByteArray); - - return worldCompressionModeByteArray; + try + { + compressedIn.readFully(outputByteArray.elements(), 0, FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH); + } + catch (EOFException e) + { + throw new DataCorruptedException(e); + } } - private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static void writeDataMappingToBlob(FullDataPointIdMap mapping, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum); @@ -350,12 +336,11 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject compressedOut.flush(); byteArrayOutputStream.close(); - - return byteArrayOutputStream.toByteArray(); + outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); } - private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, long pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException + private static FullDataPointIdMap readBlobToDataMapping(ByteArrayList compressedMappingByteArray, long pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray.toByteArray()); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper); @@ -371,27 +356,36 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject @Override public void encode(ByteBuf out) { - out.writeLong(this.pos); - out.writeInt(this.dataChecksum); - - out.writeInt(this.compressedDataByteArray.length); - out.writeBytes(this.compressedDataByteArray); - - out.writeInt(this.compressedColumnGenStepByteArray.length); - out.writeBytes(this.compressedColumnGenStepByteArray); - out.writeInt(this.compressedWorldCompressionModeByteArray.length); - out.writeBytes(this.compressedWorldCompressionModeByteArray); - - out.writeInt(this.compressedMappingByteArray.length); - out.writeBytes(this.compressedMappingByteArray); - - out.writeByte(this.dataFormatVersion); - out.writeByte(this.compressionModeValue); - - out.writeBoolean(this.applyToParent); - - out.writeLong(this.lastModifiedUnixDateTime); - out.writeLong(this.createdUnixDateTime); + try + { + out.writeLong(this.pos); + out.writeInt(this.dataChecksum); + + out.writeInt(this.compressedDataByteArray.size()); + // writing a stream can throw an IO Exception, however since this is just a array wrapper + // nothing should be thrown unless we have the size wrong + out.writeBytes(new ByteArrayInputStream(this.compressedDataByteArray.elements()), this.compressedDataByteArray.size()); + + out.writeInt(this.compressedColumnGenStepByteArray.size()); + out.writeBytes(new ByteArrayInputStream(this.compressedColumnGenStepByteArray.elements()), this.compressedColumnGenStepByteArray.size()); + out.writeInt(this.compressedWorldCompressionModeByteArray.size()); + out.writeBytes(new ByteArrayInputStream(this.compressedWorldCompressionModeByteArray.elements()), this.compressedWorldCompressionModeByteArray.size()); + + out.writeInt(this.compressedMappingByteArray.size()); + out.writeBytes(new ByteArrayInputStream(this.compressedMappingByteArray.elements()), this.compressedMappingByteArray.size()); + + out.writeByte(this.dataFormatVersion); + out.writeByte(this.compressionModeValue); + + out.writeBoolean(this.applyToParent); + + out.writeLong(this.lastModifiedUnixDateTime); + out.writeLong(this.createdUnixDateTime); + } + catch (IOException e) + { + throw new RuntimeException(e); + } } @Override @@ -400,16 +394,16 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject this.pos = in.readLong(); this.dataChecksum = in.readInt(); - this.compressedDataByteArray = new byte[in.readInt()]; - in.readBytes(this.compressedDataByteArray); + ListUtil.clearAndSetSize(this.compressedDataByteArray, in.readInt()); + in.readBytes(this.compressedDataByteArray.elements()); - this.compressedColumnGenStepByteArray = new byte[in.readInt()]; - in.readBytes(this.compressedColumnGenStepByteArray); - this.compressedWorldCompressionModeByteArray = new byte[in.readInt()]; - in.readBytes(this.compressedWorldCompressionModeByteArray); + ListUtil.clearAndSetSize(this.compressedColumnGenStepByteArray, in.readInt()); + in.readBytes(this.compressedColumnGenStepByteArray.elements()); + ListUtil.clearAndSetSize(this.compressedWorldCompressionModeByteArray, in.readInt()); + in.readBytes(this.compressedWorldCompressionModeByteArray.elements()); - this.compressedMappingByteArray = new byte[in.readInt()]; - in.readBytes(this.compressedMappingByteArray); + ListUtil.clearAndSetSize(this.compressedMappingByteArray, in.readInt()); + in.readBytes(this.compressedMappingByteArray.elements()); this.dataFormatVersion = in.readByte(); this.compressionModeValue = in.readByte(); @@ -422,6 +416,14 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject + //================// + // helper methods // + //================// + + public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } + + + //===========// // overrides // //===========// @@ -438,10 +440,10 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject .add("levelMinY", this.levelMinY) .add("pos", this.pos) .add("dataChecksum", this.dataChecksum) - .add("compressedDataByteArray length", this.compressedDataByteArray.length) - .add("compressedColumnGenStepByteArray length", this.compressedColumnGenStepByteArray.length) - .add("compressedWorldCompressionModeByteArray length", this.compressedWorldCompressionModeByteArray.length) - .add("compressedMappingByteArray length", this.compressedMappingByteArray.length) + .add("compressedDataByteArray length", this.compressedDataByteArray.size()) + .add("compressedColumnGenStepByteArray length", this.compressedColumnGenStepByteArray.size()) + .add("compressedWorldCompressionModeByteArray length", this.compressedWorldCompressionModeByteArray.size()) + .add("compressedMappingByteArray length", this.compressedMappingByteArray.size()) .add("dataFormatVersion", this.dataFormatVersion) .add("compressionModeValue", this.compressionModeValue) .add("applyToParent", this.applyToParent) @@ -450,10 +452,6 @@ public class FullDataSourceV2DTO implements IBaseDTO, INetworkObject .toString(); } - //================// - // helper methods // - //================// - public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java index 3d9ebba20..5b2926696 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java @@ -23,11 +23,14 @@ package com.seibel.distanthorizons.core.sql.dto; * DTO = DataTable Object
* Any object that's stored in the database should extend this object. */ -public interface IBaseDTO +public interface IBaseDTO extends AutoCloseable { TKey getKey(); /** Can be used for keys that don't have a clean human readable toString() method. */ default String getKeyDisplayString() { return this.getKey().toString(); } + // closing a DTO shouldn't have the possibility of throwing anything + @Override + void close(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index 88e5cda02..8e0b08683 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.DatabaseUpdater; import com.seibel.distanthorizons.core.sql.DbConnectionClosedException; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; @@ -179,13 +180,20 @@ public abstract class AbstractDhRepo> implemen public TDTO getByKey(TKey primaryKey) { - Map objectMap = this.queryDictionaryFirst(this.createSelectByKeySql(primaryKey)); - if (objectMap != null && !objectMap.isEmpty()) + try(ResultSet resultSet = this.query(this.createSelectStatementByKey(primaryKey))) { - return this.convertDictionaryToDto(objectMap); + if (resultSet != null && resultSet.next()) + { + return this.convertResultSetToDto(resultSet); + } + else + { + return null; + } } - else + catch (SQLException | IOException e) { + LOGGER.warn("Unexpected issue deserializing DTO ["+this.dtoClass.getSimpleName()+"] with primary key ["+primaryKey+"]. Error: ["+e.getMessage()+"].", e); return null; } } @@ -228,7 +236,7 @@ public abstract class AbstractDhRepo> implemen } catch (SQLException e) { - String message = "Unexpected insert statement error: ["+e.getMessage()+"]."; + String message = "Unexpected DTO insert error: ["+e.getMessage()+"]."; LOGGER.error(message); throw new RuntimeException(message, e); } @@ -245,7 +253,7 @@ public abstract class AbstractDhRepo> implemen } catch (SQLException e) { - String message = "Unexpected update statement error: ["+e.getMessage()+"]."; + String message = "Unexpected DTO update error: ["+e.getMessage()+"]."; LOGGER.error(message); throw new RuntimeException(message, e); } @@ -255,55 +263,67 @@ public abstract class AbstractDhRepo> implemen public void delete(TDTO dto) { this.deleteWithKey(dto.getKey()); } public void deleteWithKey(TKey key) { - String whereEqualStatement = this.createWhereStatement(key); - this.queryDictionaryFirst("DELETE FROM "+this.getTableName()+" WHERE "+whereEqualStatement); + try (PreparedStatement statement = this.createDeleteStatementByKey(key)) + { + this.query(statement); + } + catch (SQLException e) + { + throw new RuntimeException(e); + } } /** With great power comes great responsibility... */ - public void deleteAll() { this.queryDictionaryFirst("DELETE FROM "+this.getTableName()); } + public void deleteAll() + { + String sql = "DELETE FROM " + this.getTableName(); + try (PreparedStatement statement = this.createPreparedStatement(sql)) + { + this.query(statement); + } + catch (SQLException e) + { + throw new RuntimeException(e); + } + } public boolean exists(TDTO dto) { return this.existsWithKey(dto.getKey()); } public boolean existsWithKey(TKey key) { - String whereEqualStatement = this.createWhereStatement(key); - Map result = this.queryDictionaryFirst("SELECT EXISTS(SELECT 1 FROM "+this.getTableName()+" WHERE "+whereEqualStatement+") as 'existingCount';"); - return result != null && (int)result.get("existingCount") != 0; + try + { + try (PreparedStatement statement = this.createExistsStatementByKey(key); + ResultSet result = this.query(statement)) + { + return result != null && result.getInt("existingCount") != 0; + } + } + catch (SQLException e) + { + return false; + } } + //==============// // low level DB // //==============// - public List> queryDictionary(String sql) - { - try - { - return this.query(sql); - } - catch (DbConnectionClosedException e) - { - return new ArrayList<>(); - } - } - public List> queryDictionary(PreparedStatement preparedStatement) - { - try - { - return this.query(preparedStatement); - } - catch (DbConnectionClosedException e) - { - return new ArrayList<>(); - } - } + /** + * This can only run 1 command at a time.

+ * + * Note: {@link AbstractDhRepo#query(PreparedStatement)} with a {@link PreparedStatement} + * should be used if the query will be run often. + * This reduces GC pressure due to the {@link String} and {@link Map} allocation cost. + */ @Nullable public Map queryDictionaryFirst(String sql) { try { - List> objectList = this.query(sql); + List> objectList = this.queryDictionary(sql); return !objectList.isEmpty() ? objectList.get(0) : null; } catch (DbConnectionClosedException e) @@ -311,54 +331,14 @@ public abstract class AbstractDhRepo> implemen return null; } } - @Nullable - public Map queryDictionaryFirst(PreparedStatement preparedStatement) - { - try - { - List> objectList = this.query(preparedStatement); - return !objectList.isEmpty() ? objectList.get(0) : null; - } - catch (DbConnectionClosedException e) - { - return null; - } - } - - - /** note: this can only handle 1 command at a time */ - private List> query(PreparedStatement statement) throws RuntimeException, DbConnectionClosedException - { - try - { - statement.setQueryTimeout(TIMEOUT_SECONDS); - - // Note: this can only handle 1 command at a time - boolean resultSetPresent = statement.execute(); - try (ResultSet resultSet = statement.getResultSet()) - { - return this.parseQueryResult(resultSet, resultSetPresent); - } - } - catch(SQLException e) - { - // SQL exceptions generally only happen when something is wrong with - // the database or the query and should cause the system to blow up to notify the developer - - if (DbConnectionClosedException.IsClosedException(e)) - { - throw new DbConnectionClosedException(e); - } - else - { - String message = "Unexpected Query error: [" + e.getMessage() + "], for prepared statement: [" + statement + "]."; - LOGGER.error(message); - throw new RuntimeException(message, e); - } - } - } - /** note: this can only handle 1 command at a time */ - private List> query(String sql) throws RuntimeException, DbConnectionClosedException + /** + * This can only run 1 command at a time.

+ * + * Note: {@link AbstractDhRepo#query(PreparedStatement)} with a {@link PreparedStatement} + * should be used if the query will be run often. + * This reduces GC pressure due to the {@link String} and {@link Map} allocation cost. + */ + private List> queryDictionary(String sql) throws RuntimeException, DbConnectionClosedException { try (Statement statement = this.connection.createStatement()) { @@ -368,7 +348,7 @@ public abstract class AbstractDhRepo> implemen boolean resultSetPresent = statement.execute(sql); try (ResultSet resultSet = statement.getResultSet()) { - return this.parseQueryResult(resultSet, resultSetPresent); + return this.convertResultSetToDictionaryList(resultSet, resultSetPresent); } } catch(SQLException e) @@ -383,32 +363,71 @@ public abstract class AbstractDhRepo> implemen else { String message = "Unexpected Query error: [" + e.getMessage() + "], for script: [" + sql + "]."; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } + } + } + + + + /** + * Warning: both the returned {@link ResultSet} and incoming {@link PreparedStatement} + * must be wrapped in a try-resource block to prevent memory + * leaks and issues with the DB becoming locked. + */ + @Nullable + public ResultSet query(@Nullable PreparedStatement statement) throws RuntimeException + { + // This is done so we don't have to add "if null" checks everywhere. + // Normally this should only happen once the DB has been closed. + if (statement == null) + { + return null; + } + + + try + { + statement.setQueryTimeout(TIMEOUT_SECONDS); + + // Note: this can only handle 1 command at a time + boolean resultSetPresent = statement.execute(); + if (resultSetPresent) + { + return statement.getResultSet(); + } + else + { + return null; + } + } + catch(SQLException e) + { + // SQL exceptions generally only happen when something is wrong with + // the database or the query and should cause the system to blow up to notify the developer + + if (DbConnectionClosedException.IsClosedException(e)) + { + return null; + } + else + { + String message = "Unexpected Query error: [" + e.getMessage() + "], for prepared statement: [" + statement + "]."; LOGGER.error(message); throw new RuntimeException(message, e); } } } - private List> parseQueryResult(ResultSet resultSet, boolean resultSetPresent) throws SQLException - { - if (resultSetPresent) - { - List> resultList = convertResultSetToDictionaryList(resultSet); - resultSet.close(); - return resultList; - } - else - { - if (resultSet != null) - { - resultSet.close(); - } - - return new ArrayList<>(); - } - } - public PreparedStatement createPreparedStatement(String sql) throws DbConnectionClosedException + + /** + * @return Null if the database was closed + * @throws RuntimeException if there was a problem with the given SQL string + */ + @Nullable + public PreparedStatement createPreparedStatement(String sql) throws RuntimeException { try { @@ -420,7 +439,7 @@ public abstract class AbstractDhRepo> implemen { if (DbConnectionClosedException.IsClosedException(e)) { - throw new DbConnectionClosedException(e); + return null; } else { @@ -525,9 +544,25 @@ public abstract class AbstractDhRepo> implemen // helper methods // //================// - public String createWhereStatement(TDTO dto) { return this.createWhereStatement(dto.getKey()); } - - public static List> convertResultSetToDictionaryList(ResultSet resultSet) throws SQLException + private List> convertResultSetToDictionaryList(ResultSet resultSet, boolean resultSetPresent) throws SQLException + { + if (resultSetPresent) + { + List> resultList = convertResultSetToDictionaryList(resultSet); + resultSet.close(); + return resultList; + } + else + { + if (resultSet != null) + { + resultSet.close(); + } + + return new ArrayList<>(); + } + } + private static List> convertResultSetToDictionaryList(ResultSet resultSet) throws SQLException { List> list = new ArrayList<>(); @@ -540,7 +575,7 @@ public abstract class AbstractDhRepo> implemen for (int columnIndex = 1; columnIndex <= resultColumnCount; columnIndex++) // column indices start at 1 { String columnName = resultMetaData.getColumnName(columnIndex); - if (columnName == null || columnName.equals("")) + if (columnName == null || columnName.isEmpty()) { throw new RuntimeException("SQL result set is missing a column name for column ["+resultMetaData.getTableName(columnIndex)+"."+columnIndex+"]."); } @@ -585,18 +620,85 @@ public abstract class AbstractDhRepo> implemen public abstract String getTableName(); @Nullable - public abstract TDTO convertDictionaryToDto(Map objectMap) throws ClassCastException; + public abstract TDTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, IOException, SQLException; + + - public String createSelectByKeySql(TKey key) { return "SELECT * FROM "+this.getTableName()+" WHERE "+this.createWhereStatement(key); } /** - * Example: - * Id = '0' - * ColOne = '0' AND ColTwo = '2' + * should NOT start with WHERE + * Example: TODO */ - public abstract String createWhereStatement(TKey key); + protected abstract String CreateParameterizedWhereString(); + protected void setPreparedStatementWhereClause(PreparedStatement statement, TKey key) throws SQLException { this.setPreparedStatementWhereClause(statement, 1, key); } + protected abstract int setPreparedStatementWhereClause(PreparedStatement statement, int parameterIndex, TKey key) throws SQLException; + + + private String selectSqlTemplate = null; + public PreparedStatement createSelectStatementByKey(TKey key) throws SQLException + { + // create shared template string + if (this.selectSqlTemplate == null) + { + this.selectSqlTemplate = "SELECT * FROM "+this.getTableName() + " WHERE " + this.CreateParameterizedWhereString(); + } + + PreparedStatement statement = this.createPreparedStatement(this.selectSqlTemplate); + if (statement == null) + { + return null; + } + this.setPreparedStatementWhereClause(statement, key); + + return statement; + } + + private String existsSqlTemplate = null; + public PreparedStatement createExistsStatementByKey(TKey key) throws SQLException + { + // create shared template string + if (this.existsSqlTemplate == null) + { + this.existsSqlTemplate = "SELECT EXISTS(SELECT 1 FROM "+this.getTableName()+" WHERE "+this.CreateParameterizedWhereString()+") as 'existingCount'"; + } + + PreparedStatement statement = this.createPreparedStatement(this.existsSqlTemplate); + if (statement == null) + { + return null; + } + this.setPreparedStatementWhereClause(statement, key); + + return statement; + } + + private String deleteSqlTemplate = null; + public PreparedStatement createDeleteStatementByKey(TKey key) throws SQLException + { + // create shared template string + if (this.deleteSqlTemplate == null) + { + this.deleteSqlTemplate = "DELETE FROM "+this.getTableName()+" WHERE " + this.CreateParameterizedWhereString(); + } + + PreparedStatement statement = this.createPreparedStatement(this.deleteSqlTemplate); + if (statement == null) + { + return null; + } + this.setPreparedStatementWhereClause(statement, key); + + return statement; + } + + + + @Nullable public abstract PreparedStatement createInsertStatement(TDTO dto) throws SQLException; + @Nullable public abstract PreparedStatement createUpdateStatement(TDTO dto) throws SQLException; + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/BeaconBeamRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/BeaconBeamRepo.java index 3f5951a94..4a0a26be0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/BeaconBeamRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/BeaconBeamRepo.java @@ -26,10 +26,12 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO; import com.seibel.distanthorizons.core.util.LodUtil; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; import java.awt.*; import java.io.File; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -60,7 +62,16 @@ public class BeaconBeamRepo extends AbstractDhRepo public String getTableName() { return "BeaconBeam"; } @Override - public String createWhereStatement(DhBlockPos pos) { return "BlockPosX = "+ pos.getX() +" AND BlockPosY = "+ pos.getY() +" AND BlockPosZ = "+ pos.getZ(); } + protected String CreateParameterizedWhereString() { return "BlockPosX = ? AND BlockPosY = ? AND BlockPosZ = ?"; } + + @Override + protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, DhBlockPos pos) throws SQLException + { + statement.setInt(index++, pos.getX()); + statement.setInt(index++, pos.getY()); + statement.setInt(index++, pos.getZ()); + return index; + } @@ -69,15 +80,16 @@ public class BeaconBeamRepo extends AbstractDhRepo //=======================// @Override - public BeaconBeamDTO convertDictionaryToDto(Map objectMap) throws ClassCastException + @Nullable + public BeaconBeamDTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, SQLException { - int posX = (Integer) objectMap.get("BlockPosX"); - int posY = (Integer) objectMap.get("BlockPosY"); - int posZ = (Integer) objectMap.get("BlockPosZ"); + int posX = resultSet.getInt("BlockPosX"); + int posY = resultSet.getInt("BlockPosY"); + int posZ = resultSet.getInt("BlockPosZ"); - int red = (Integer) objectMap.get("ColorR"); - int green = (Integer) objectMap.get("ColorG"); - int blue = (Integer) objectMap.get("ColorB"); + int red = resultSet.getInt("ColorR"); + int green = resultSet.getInt("ColorG"); + int blue = resultSet.getInt("ColorB"); BeaconBeamDTO dto = new BeaconBeamDTO(new DhBlockPos(posX, posY, posZ), new Color(red, green, blue)); @@ -98,18 +110,23 @@ public class BeaconBeamRepo extends AbstractDhRepo " ?, ? \n" + ");"; PreparedStatement statement = this.createPreparedStatement(sql); + if (statement == null) + { + return null; + } + int i = 1; - statement.setObject(i++, dto.blockPos.getX()); - statement.setObject(i++, dto.blockPos.getY()); - statement.setObject(i++, dto.blockPos.getZ()); + statement.setInt(i++, dto.blockPos.getX()); + statement.setInt(i++, dto.blockPos.getY()); + statement.setInt(i++, dto.blockPos.getZ()); - statement.setObject(i++, dto.color.getRed()); - statement.setObject(i++, dto.color.getGreen()); - statement.setObject(i++, dto.color.getBlue()); + statement.setInt(i++, dto.color.getRed()); + statement.setInt(i++, dto.color.getGreen()); + statement.setInt(i++, dto.color.getBlue()); - statement.setObject(i++, System.currentTimeMillis()); // last modified unix time - statement.setObject(i++, System.currentTimeMillis()); // created unix time + statement.setLong(i++, System.currentTimeMillis()); // last modified unix time + statement.setLong(i++, System.currentTimeMillis()); // created unix time return statement; } @@ -124,17 +141,21 @@ public class BeaconBeamRepo extends AbstractDhRepo " LastModifiedUnixDateTime = ? \n" + "WHERE BlockPosX = ? AND BlockPosY = ? AND BlockPosZ = ?"; PreparedStatement statement = this.createPreparedStatement(sql); + if (statement == null) + { + return null; + } int i = 1; - statement.setObject(i++, dto.color.getRed()); - statement.setObject(i++, dto.color.getGreen()); - statement.setObject(i++, dto.color.getBlue()); + statement.setInt(i++, dto.color.getRed()); + statement.setInt(i++, dto.color.getGreen()); + statement.setInt(i++, dto.color.getBlue()); - statement.setObject(i++, System.currentTimeMillis()); // last modified unix time + statement.setLong(i++, System.currentTimeMillis()); // last modified unix time - statement.setObject(i++, dto.blockPos.getX()); - statement.setObject(i++, dto.blockPos.getY()); - statement.setObject(i++, dto.blockPos.getZ()); + statement.setInt(i++, dto.blockPos.getX()); + statement.setInt(i++, dto.blockPos.getY()); + statement.setInt(i++, dto.blockPos.getZ()); return statement; } @@ -171,22 +192,44 @@ public class BeaconBeamRepo extends AbstractDhRepo ); } + private final String getAllBeamsInRangeTemplate = + "SELECT * " + + "FROM "+this.getTableName()+" " + + "WHERE " + + "? <= BlockPosX AND BlockPosX <= ? AND " + + "? <= BlockPosZ AND BlockPosZ <= ?"; public List getAllBeamsInBlockPosRange( int minBlockX, int minBlockZ, int maxBlockX, int maxBlockZ ) { - List> objectMapList = this.queryDictionary( - "SELECT * " + - "FROM "+this.getTableName()+" " + - "WHERE " + - minBlockX+" <= BlockPosX AND BlockPosX <= "+maxBlockX+" AND " + - minBlockZ+" <= BlockPosZ AND BlockPosZ <= "+maxBlockZ); - ArrayList beamList = new ArrayList<>(); - for (Map objectMap : objectMapList) + + try(PreparedStatement statement = this.createPreparedStatement(this.getAllBeamsInRangeTemplate)) { - beamList.add(this.convertDictionaryToDto(objectMap)); + if(statement == null) + { + return beamList; + } + + int i = 1; + statement.setInt(i++, minBlockX); + statement.setInt(i++, minBlockZ); + statement.setInt(i++, maxBlockX); + statement.setInt(i++, maxBlockZ); + + + try (ResultSet result = this.query(statement)) + { + while (result != null && result.next()) + { + beamList.add(this.convertResultSetToDto(result)); + } + } + } + catch (Exception e) + { + throw new RuntimeException(e); } return beamList; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/ChunkHashRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/ChunkHashRepo.java index d3c583fdb..12ceabdb1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/ChunkHashRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/ChunkHashRepo.java @@ -21,11 +21,14 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO; import com.seibel.distanthorizons.core.sql.dto.ChunkHashDTO; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; import java.io.File; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -54,7 +57,15 @@ public class ChunkHashRepo extends AbstractDhRepo public String getTableName() { return "ChunkHash"; } @Override - public String createWhereStatement(DhChunkPos pos) { return "ChunkPosX = '"+ pos.getX() +"' AND ChunkPosZ = '"+ pos.getZ() +"'"; } + protected String CreateParameterizedWhereString() { return "ChunkPosX = ? AND ChunkPosZ = ?"; } + + @Override + protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, DhChunkPos pos) throws SQLException + { + statement.setInt(index++, pos.getX()); + statement.setInt(index++, pos.getZ()); + return index; + } @@ -62,13 +73,14 @@ public class ChunkHashRepo extends AbstractDhRepo // repo required methods // //=======================// - @Override - public ChunkHashDTO convertDictionaryToDto(Map objectMap) throws ClassCastException + @Override + @Nullable + public ChunkHashDTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, SQLException { - int posX = (Integer) objectMap.get("ChunkPosX"); - int posZ = (Integer) objectMap.get("ChunkPosZ"); + int posX = resultSet.getInt("ChunkPosX"); + int posZ = resultSet.getInt("ChunkPosZ"); - int chunkHash = (Integer) objectMap.get("ChunkHash"); + int chunkHash = resultSet.getInt("ChunkHash"); ChunkHashDTO dto = new ChunkHashDTO(new DhChunkPos(posX, posZ), chunkHash); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index a61bc22ab..449344fcc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.dto.ChunkHashDTO; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.coreapi.util.StringUtil; import it.unimi.dsi.fastutil.longs.LongArrayList; @@ -28,6 +29,7 @@ import org.jetbrains.annotations.Nullable; import java.io.File; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -58,7 +60,14 @@ public class FullDataSourceV1Repo extends AbstractDhRepo objectMap) throws ClassCastException + @Override + @Nullable + public FullDataSourceV1DTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, SQLException { - String posString = (String) objectMap.get("DhSectionPos"); + String posString = resultSet.getString("DhSectionPos"); Long pos = deserializeSectionPos(posString); // meta data - int checksum = (Integer) objectMap.get("Checksum"); - byte dataDetailLevel = (Byte) objectMap.get("DataDetailLevel"); - String worldGenStepString = (String) objectMap.get("WorldGenStep"); + int checksum = resultSet.getInt("Checksum"); + byte dataDetailLevel = resultSet.getByte("DataDetailLevel"); + String worldGenStepString = resultSet.getString("WorldGenStep"); EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromName(worldGenStepString); - String dataType = (String) objectMap.get("DataType"); - byte binaryDataFormatVersion = (Byte) objectMap.get("BinaryDataFormatVersion"); + String dataType = resultSet.getString("DataType"); + byte binaryDataFormatVersion = resultSet.getByte("BinaryDataFormatVersion"); // binary data - byte[] dataByteArray = (byte[]) objectMap.get("Data"); + byte[] dataByteArray = resultSet.getBytes("Data"); FullDataSourceV1DTO dto = new FullDataSourceV1DTO( pos, @@ -92,41 +102,43 @@ public class FullDataSourceV1Repo extends AbstractDhRepo - * Returns {@link DhSectionPos#SECTION_MINIMUM_DETAIL_LEVEL} if no data is present. - */ - public int getMaxSectionDetailLevel() - { - Map resultMap = this.queryDictionaryFirst("select MAX(DataDetailLevel) as maxDetailLevel from "+this.getTableName()+";"); - int maxDetailLevel; - if (resultMap == null || resultMap.get("maxDetailLevel") == null) - { - maxDetailLevel = 0; - } - else - { - maxDetailLevel = (int)resultMap.get("maxDetailLevel"); - } - - return maxDetailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - } - - - //===========// // migration // //===========// @@ -207,25 +201,47 @@ public class FullDataSourceV1Repo extends AbstractDhRepo 1 " + + "LIMIT ?;"; /** Returns the new "returnCount" positions that need to be migrated */ public LongArrayList getPositionsToMigrate(int returnCount) { - LongArrayList list = new LongArrayList(); + LongArrayList posList = new LongArrayList(); - List> resultMapList = this.queryDictionary( - "select DhSectionPos " + - "from "+this.getTableName()+" " + - "WHERE MigrationFailed <> 1 " + - "LIMIT "+returnCount+";"); - - for (Map resultMap : resultMapList) + try(PreparedStatement statement = this.createPreparedStatement(this.getMigrationPositionsSqlTemplate)) { - // returned in the format [sectionDetailLevel,x,z] IE [6,0,0] - long sectionPos = deserializeSectionPos((String) resultMap.get("DhSectionPos")); - list.add(sectionPos); + if (statement == null) + { + return posList; + } + + + int i = 1; + statement.setInt(i++, returnCount); + + try (ResultSet result = this.query(statement)) + { + while (result != null && result.next()) + { + String posString = result.getString("DhSectionPos"); + // returned in the format [sectionDetailLevel,x,z] IE [6,0,0] + Long sectionPos = deserializeSectionPos(posString); + if (sectionPos != null) + { + posList.add(sectionPos.longValue()); + } + } + } + } + catch (Exception e) + { + throw new RuntimeException(e); } - return list; + return posList; } public void markMigrationFailed(long pos) @@ -263,17 +279,38 @@ public class FullDataSourceV1Repo extends AbstractDhRepo 0 OR DataType <> 'CompleteFullDataSource' " + + "LIMIT ?"; /** Returns single quote surrounded {@link DhSectionPos} serailzed values */ - public ArrayList getUnusedDataSourcePositionStringList(int deleteCount) + public ArrayList getUnusedDataSourcePositionStringList(int limit) { - List> deletePosResultMapList = this.queryDictionary( - "select DhSectionPos from "+this.getTableName()+" where DataDetailLevel <> 0 or DataType <> 'CompleteFullDataSource' limit "+deleteCount); - ArrayList deletePosList = new ArrayList<>(); - for (Map deletePosMap : deletePosResultMapList) + + try(PreparedStatement statement = this.createPreparedStatement(this.getUnusedPositionSqlTemplate)) { - String posString = (String) deletePosMap.get("DhSectionPos"); - deletePosList.add("'"+posString+"'"); + if (statement == null) + { + return deletePosList; + } + + int i = 1; + statement.setInt(i++, limit); + + try (ResultSet result = this.query(statement)) + { + while (result != null && result.next()) + { + String posString = result.getString("DhSectionPos"); + deletePosList.add("'"+posString+"'"); + } + } + } + catch (SQLException e) + { + throw new RuntimeException(e); } return deletePosList; @@ -294,7 +331,6 @@ public class FullDataSourceV1Repo extends AbstractDhRepo { @@ -65,71 +63,98 @@ public class FullDataSourceV2Repo extends AbstractDhRepo objectMap) throws ClassCastException + @Nullable + public FullDataSourceV2DTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, IOException, SQLException { - byte detailLevel = (Byte) objectMap.get("DetailLevel"); + //======================// + // get statement values // + //======================// + + byte detailLevel = resultSet.getByte("DetailLevel"); byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - int posX = (Integer) objectMap.get("PosX"); - int posZ = (Integer) objectMap.get("PosZ"); + int posX = resultSet.getInt("PosX"); + int posZ = resultSet.getInt("PosZ"); long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ); - int minY = (Integer) objectMap.get("MinY"); - int dataChecksum = (Integer) objectMap.get("DataChecksum"); - - byte[] dataByteArray = (byte[]) objectMap.get("Data"); - byte[] columnGenStepByteArray = (byte[]) objectMap.get("ColumnGenerationStep"); - byte[] columnWorldCompressionByteArray = (byte[]) objectMap.get("ColumnWorldCompressionMode"); - byte[] mappingByteArray = (byte[]) objectMap.get("Mapping"); + int minY = resultSet.getInt("MinY"); + int dataChecksum = resultSet.getInt("DataChecksum"); - byte dataFormatVersion = (Byte) objectMap.get("DataFormatVersion"); - byte compressionModeValue = (Byte) objectMap.get("CompressionMode"); + byte dataFormatVersion = resultSet.getByte("DataFormatVersion"); + byte compressionModeValue = resultSet.getByte("CompressionMode"); - boolean applyToParent = ((int) objectMap.get("ApplyToParent")) == 1; + boolean applyToParent = (resultSet.getInt("ApplyToParent")) == 1; - long lastModifiedUnixDateTime = (Long) objectMap.get("LastModifiedUnixDateTime"); - long createdUnixDateTime = (Long) objectMap.get("CreatedUnixDateTime"); + long lastModifiedUnixDateTime = resultSet.getLong("LastModifiedUnixDateTime"); + long createdUnixDateTime = resultSet.getLong("CreatedUnixDateTime"); - FullDataSourceV2DTO dto = new FullDataSourceV2DTO( - pos, - dataChecksum, columnGenStepByteArray, columnWorldCompressionByteArray, dataFormatVersion, compressionModeValue, dataByteArray, - lastModifiedUnixDateTime, createdUnixDateTime, - mappingByteArray, applyToParent, - minY); + + + //===================// + // set DTO variables // + //===================// + + FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding(); + + // set pooled arrays + dto.compressedDataByteArray = putAllBytes(resultSet.getBinaryStream("Data"), dto.compressedDataByteArray); + dto.compressedColumnGenStepByteArray = putAllBytes(resultSet.getBinaryStream("ColumnGenerationStep"), dto.compressedColumnGenStepByteArray); + dto.compressedWorldCompressionModeByteArray = putAllBytes(resultSet.getBinaryStream("ColumnWorldCompressionMode"), dto.compressedWorldCompressionModeByteArray); + dto.compressedMappingByteArray = putAllBytes(resultSet.getBinaryStream("Mapping"), dto.compressedMappingByteArray); + + // set individual variables + { + dto.pos = pos; + dto.dataChecksum = dataChecksum; + dto.dataFormatVersion = dataFormatVersion; + dto.compressionModeValue = compressionModeValue; + dto.lastModifiedUnixDateTime = lastModifiedUnixDateTime; + dto.createdUnixDateTime = createdUnixDateTime; + dto.applyToParent = applyToParent; + dto.levelMinY = minY; + } return dto; } + private final String insertSqlTemplate = + "INSERT INTO "+this.getTableName() + " (\n" + + " DetailLevel, PosX, PosZ, \n" + + " MinY, DataChecksum, \n" + + " Data, ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" + + " DataFormatVersion, CompressionMode, ApplyToParent, \n" + + " LastModifiedUnixDateTime, CreatedUnixDateTime) \n" + + "VALUES( \n" + + " ?, ?, ?, \n" + + " ?, ?, \n" + + " ?, ?, ?, ?, \n" + + " ?, ?, ?, \n" + + " ?, ? \n" + + ");"; @Override public PreparedStatement createInsertStatement(FullDataSourceV2DTO dto) throws SQLException { - String sql = - "INSERT INTO "+this.getTableName() + " (\n" + - " DetailLevel, PosX, PosZ, \n" + - " MinY, DataChecksum, \n" + - " Data, ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" + - " DataFormatVersion, CompressionMode, ApplyToParent, \n" + - " LastModifiedUnixDateTime, CreatedUnixDateTime) \n" + - "VALUES( \n" + - " ?, ?, ?, \n" + - " ?, ?, \n" + - " ?, ?, ?, ?, \n" + - " ?, ?, ?, \n" + - " ?, ? \n" + - ");"; - PreparedStatement statement = this.createPreparedStatement(sql); + PreparedStatement statement = this.createPreparedStatement(this.insertSqlTemplate); + if (statement == null) + { + return null; + } + int i = 1; statement.setObject(i++, DhSectionPos.getDetailLevel(dto.pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); @@ -139,10 +164,10 @@ public class FullDataSourceV2Repo extends AbstractDhRepo> resultMapList = this.queryDictionary( - "SELECT DetailLevel, PosX, PosZ, " + - "(sqrt(pow(PosX - "+targetBlockPosX+", 2) + pow(PosZ - "+targetBlockPosZ+", 2))) AS Distance " + - "FROM "+this.getTableName()+" " + - "WHERE ApplyToParent = 1 " + - "ORDER BY Distance ASC " + - "LIMIT "+returnCount+"; " - ); - for (Map resultMap : resultMapList) + PreparedStatement statement = this.createPreparedStatement(this.getPositionsToUpdateSql); + if (statement == null) { - byte detailLevel = (Byte) resultMap.get("DetailLevel"); - byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - int posX = (Integer) resultMap.get("PosX"); - int posZ = (Integer) resultMap.get("PosZ"); - - long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ); - list.add(pos); + return list; } - return list; + try + { + int i = 1; + statement.setInt(i++, targetBlockPosX); + statement.setInt(i++, targetBlockPosZ); + + statement.setInt(i++, returnCount); + + ResultSet result = this.query(statement); + while (result != null && result.next()) + { + byte detailLevel = result.getByte("DetailLevel"); + byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + int posX = result.getInt("PosX"); + int posZ = result.getInt("PosZ"); + + long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ); + list.add(pos); + } + + return list; + } + catch (SQLException e) + { + throw new RuntimeException(e); + } } + + private final String getColumnGenerationStepSql = + "select ColumnGenerationStep, CompressionMode " + + "from "+this.getTableName()+" " + + "WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?"; /** @return null if nothing exists for this position */ - public byte[] getColumnGenerationStepForPos(long pos) + public void getColumnGenerationStepForPos(long pos, ByteArrayList outputByteArray) { - int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - - Map resultMap = this.queryDictionaryFirst( - "select ColumnGenerationStep, CompressionMode " + - "from "+this.getTableName()+" " + - "WHERE DetailLevel = "+detailLevel+" AND PosX = "+ DhSectionPos.getX(pos)+" AND PosZ = "+ DhSectionPos.getZ(pos)); - - if (resultMap != null) + PreparedStatement statement = this.createPreparedStatement(this.getColumnGenerationStepSql); + if (statement == null) { - byte[] compressedByteArray = (byte[]) resultMap.get("ColumnGenerationStep"); + return; + } + + + try + { + int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - byte compressionModeEnumValue = (byte) resultMap.get("CompressionMode"); + + + int i = 1; + statement.setInt(i++, detailLevel); + statement.setInt(i++, DhSectionPos.getX(pos)); + statement.setInt(i++, DhSectionPos.getZ(pos)); + + + ResultSet result = this.query(statement); + if (result == null || !result.next()) + { + return; + } + + + byte compressionModeEnumValue = result.getByte("CompressionMode"); EDhApiDataCompressionMode compressionModeEnum = EDhApiDataCompressionMode.getFromValue(compressionModeEnumValue); try { // decompress the data - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedByteArray); - DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); - - byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; - compressedIn.readFully(columnGenStepByteArray); - - return columnGenStepByteArray; + DhDataInputStream compressedIn = new DhDataInputStream(result.getBinaryStream("ColumnGenerationStep"), compressionModeEnum); + putAllBytes(compressedIn, outputByteArray); } catch (IOException e) { LOGGER.warn("Decompression issue when getting column gen steps for pos: [" + DhSectionPos.toString(pos) + "]", e); - return null; } } - else + catch (SQLException e) { - return null; + throw new RuntimeException(e); } } @@ -292,26 +374,36 @@ public class FullDataSourceV2Repo extends AbstractDhRepo> row = this.queryDictionary(preparedStatement); - return !row.isEmpty() ? (Long) row.get(0).get("LastModifiedUnixDateTime") : null; + ResultSet result = this.query(preparedStatement); + if (result == null || !result.next()) + { + return null; + } + + return result.getLong("LastModifiedUnixDateTime"); } catch (DbConnectionClosedException e) { @@ -323,17 +415,22 @@ public class FullDataSourceV2Repo extends AbstractDhRepo getTimestampsForRange(byte detailLevel, int startPosX, int startPosZ, int endPosX, int endPosZ) { try { - PreparedStatement preparedStatement = this.createPreparedStatement( - "SELECT PosX, PosZ, LastModifiedUnixDateTime " + - "FROM " + this.getTableName() + " " + - "WHERE DetailLevel = ? " + - "AND PosX BETWEEN ? AND ? " + - "AND PosZ BETWEEN ? AND ?;" - ); + PreparedStatement preparedStatement = this.createPreparedStatement(this.getTimestampForRangeSql); + if (preparedStatement == null) + { + return new HashMap<>(); + } + int i = 1; preparedStatement.setInt(i++, detailLevel - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); @@ -342,15 +439,18 @@ public class FullDataSourceV2Repo extends AbstractDhRepo DhSectionPos.encode(detailLevel, (int) row.get("PosX"), (int) row.get("PosZ")), - row -> (long) row.get("LastModifiedUnixDateTime")) - ); - } - catch (DbConnectionClosedException e) - { - return new HashMap<>(); + + ResultSet result = this.query(preparedStatement); + HashMap returnMap = new HashMap<>(); + while (result != null && result.next()) + { + long key = DhSectionPos.encode(detailLevel, result.getInt("PosX"), result.getInt("PosZ")); + long value = result.getLong("LastModifiedUnixDateTime"); + + returnMap.put(key, value); + } + + return returnMap; } catch (SQLException e) { @@ -364,29 +464,48 @@ public class FullDataSourceV2Repo extends AbstractDhRepo> resultMapList = this.queryDictionary( - "select DetailLevel, PosX, PosZ " + - "from "+this.getTableName()+"; "); - - for (Map resultMap : resultMapList) + PreparedStatement statement = this.createPreparedStatement(getAllPositionsSql); + if (statement == null) { - byte detailLevel = (Byte) resultMap.get("DetailLevel"); - byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - int posX = (Integer) resultMap.get("PosX"); - int posZ = (Integer) resultMap.get("PosZ"); - - long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ); - list.add(pos); + return list; + } + + + try + { + ResultSet result = this.query(statement); + while (result != null && result.next()) + { + byte detailLevel = result.getByte("DetailLevel"); + byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + int posX = result.getInt("PosX"); + int posZ = result.getInt("PosZ"); + + long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ); + list.add(pos); + } + + return list; + } + catch (SQLException e) + { + throw new RuntimeException(e); } - - return list; } + + private final String getDataSizeInBytesSql = + "select LENGTH(Data) as dataSize " + + "from "+this.getTableName()+" " + + "WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?"; /** * @return the size of the full data at the given position * (doesn't include the size of the mapping or any other column) @@ -394,45 +513,91 @@ public class FullDataSourceV2Repo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( - "select LENGTH(Data) as dataSize " + - "from "+this.getTableName()+" " + - "WHERE DetailLevel = "+detailLevel+" AND PosX = "+ DhSectionPos.getX(pos)+" AND PosZ = "+ DhSectionPos.getZ(pos)); - if (resultMap != null && resultMap.get("dataSize") != null) + PreparedStatement statement = this.createPreparedStatement(this.getDataSizeInBytesSql); + if (statement == null) { - // Number cast is necessary because the returned number can be an int or long - Number resultNumber = (Number) resultMap.get("dataSize"); - long dataLength = resultNumber.longValue(); - return dataLength; - + return 0L; } - else + + try { - return 0; + int i = 1; + statement.setInt(i++, detailLevel); + statement.setInt(i++, DhSectionPos.getX(pos)); + statement.setInt(i++, DhSectionPos.getZ(pos)); + + + ResultSet result = this.query(statement); + if (result == null || !result.next()) + { + return 0L; + } + + return result.getLong("dataSize"); + } + catch (SQLException e) + { + throw new RuntimeException(e); } } + private final String getTotalDataSizeInBytesSql = + "select SUM(LENGTH(Data)) as dataSize " + + "from "+this.getTableName()+"; "; /** @return the total size in bytes of the full data for this entire database */ public long getTotalDataSizeInBytes() { - Map resultMap = this.queryDictionaryFirst( - "select SUM(LENGTH(Data)) as dataSize " + - "from "+this.getTableName()+"; "); + PreparedStatement statement = this.createPreparedStatement(this.getTotalDataSizeInBytesSql); - if (resultMap != null && resultMap.get("dataSize") != null) + try { - Number resultNumber = (Number) resultMap.get("dataSize"); - long dataLength = resultNumber.longValue(); - return dataLength; + ResultSet result = this.query(statement); + if (result == null || !result.next()) + { + return 0; + } + return result.getLong("dataSize"); } - else + catch (SQLException e) { - return 0; + throw new RuntimeException(e); } } + + //===============// + // helper method // + //===============// + + private static ByteArrayList putAllBytes(InputStream inputStream, @Nullable ByteArrayList existingArrayList) throws IOException + { + if (existingArrayList == null) + { + existingArrayList = new ByteArrayList(inputStream.available()); + } + else + { + existingArrayList.clear(); + existingArrayList.ensureCapacity(inputStream.available()); + } + + try + { + int nextByte = inputStream.read(); + while (nextByte != -1) + { + existingArrayList.add((byte) nextByte); + nextByte = inputStream.read(); + } + } + catch (EOFException ignore) { /* shouldn't happen, but just in case */ } + + return existingArrayList; + } + + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/ListUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/ListUtil.java index c3adaa220..e0598bf2e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/ListUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/ListUtil.java @@ -1,5 +1,9 @@ package com.seibel.distanthorizons.core.util; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.shorts.ShortArrayList; + import java.util.ArrayList; public class ListUtil @@ -16,4 +20,40 @@ public class ListUtil } + /** + * Unlike {@link LongArrayList#ensureCapacity(int)} this method + * will populate the list with zeros up to the given size + * so get and set methods won't cause {@link IndexOutOfBoundsException}'s. + */ + public static void clearAndSetSize(LongArrayList arrayList, int size) + { + arrayList.clear(); + for (int i = 0; i < size; i++) + { + arrayList.add(0L); + } + } + + /** @see ListUtil#clearAndSetSize(LongArrayList, int) */ + public static void clearAndSetSize(ShortArrayList arrayList, int size) + { + arrayList.clear(); + for (int i = 0; i < size; i++) + { + arrayList.add((short)0); + } + } + + /** @see ListUtil#clearAndSetSize(LongArrayList, int) */ + public static void clearAndSetSize(ByteArrayList arrayList, int size) + { + arrayList.clear(); + for (int i = 0; i < size; i++) + { + arrayList.add((byte)0); + } + } + + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java index f401cf272..59dbd6a86 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java @@ -22,8 +22,10 @@ package com.seibel.distanthorizons.core.util; import com.google.common.annotations.VisibleForTesting; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException; -import it.unimi.dsi.fastutil.longs.LongArrays; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.shorts.ShortArrayList; import it.unimi.dsi.fastutil.shorts.ShortArrays; /** @@ -43,7 +45,7 @@ import it.unimi.dsi.fastutil.shorts.ShortArrays; * * @author Builderb0y */ -public class RenderDataPointReducingList +public class RenderDataPointReducingList extends PhantomArrayListParent { /** @@ -97,7 +99,7 @@ public class RenderDataPointReducingList */ private short lowest, highest, smallest, biggest; private short sizeWithAir, sizeWithoutAir; - private final long[] links, data; + private final LongArrayList links, data; /** * a temporary array to be used for sorting nodes. * the array is first populated such that every index @@ -106,7 +108,7 @@ public class RenderDataPointReducingList * finally, the nodes are re-linked according * to the order of elements in this array. */ - private final short[] sortingArray; + private final ShortArrayList sortingArray; @@ -116,6 +118,8 @@ public class RenderDataPointReducingList public RenderDataPointReducingList(IColumnDataView view) { + super(0, 1, 2); + int size = view.size(); if (size == 0) { @@ -123,11 +127,13 @@ public class RenderDataPointReducingList this.setHighest(NULL); this.setSmallest(NULL); this.setBiggest(NULL); - this.links = LongArrays.EMPTY_ARRAY; - this.data = LongArrays.EMPTY_ARRAY; - this.sortingArray = ShortArrays.EMPTY_ARRAY; + this.links = this.pooledArraysCheckout.getLongArray(0); + this.data = this.pooledArraysCheckout.getLongArray(1); + this.sortingArray = this.pooledArraysCheckout.getShortArray(0); if (ASSERTS) this.checkLinks(); + this.pooledArraysCheckout = null; + return; } @@ -138,10 +144,13 @@ public class RenderDataPointReducingList // We will use this array for sorting the nodes, // first by lowest-to-highest, then by smallest-to-biggest. int arrayCapacity = (size << 1) - 1; - this.sortingArray = new short[arrayCapacity]; - this.links = new long[arrayCapacity]; - java.util.Arrays.fill(this.links, DEFAULT_LINKS); - this.data = new long[arrayCapacity]; + this.sortingArray = this.pooledArraysCheckout.getShortArray(0, arrayCapacity); + this.links = this.pooledArraysCheckout.getLongArray(0, arrayCapacity); + java.util.Arrays.fill(this.links.elements(), DEFAULT_LINKS); + this.data = this.pooledArraysCheckout.getLongArray(1, arrayCapacity); + + this.pooledArraysCheckout = null; + int sizeWithoutAir = 0; for (int index = 0; index < size; index++) { @@ -219,8 +228,8 @@ public class RenderDataPointReducingList throw new IllegalArgumentException(RenderDataPointUtil.toString(lowerData) + " overlaps with " + RenderDataPointUtil.toString(higherData)); } } - this.lowest = this.sortingArray[0]; - this.highest = this.sortingArray[sizeWithoutAir - 1]; + this.lowest = this.sortingArray.getShort(0); + this.highest = this.sortingArray.getShort(sizeWithoutAir - 1); // now sort by size. this.sortBySize(sizeWithAir); @@ -232,8 +241,8 @@ public class RenderDataPointReducingList this.setSmaller(biggerIndex, smallerIndex); } - this.smallest = this.sortingArray[0]; - this.biggest = this.sortingArray[sizeWithAir - 1]; + this.smallest = this.sortingArray.getShort(0); + this.biggest = this.sortingArray.getShort(sizeWithAir - 1); this.setSizeWithAir(sizeWithAir); this.setSizeWithoutAir(sizeWithoutAir); @@ -389,7 +398,7 @@ public class RenderDataPointReducingList else this.setBiggest(smaller); this.setData(index, DEFAUlT_DATA); - this.links[index] = DEFAULT_LINKS; + this.links.set(index, DEFAULT_LINKS); this.sizeWithAir--; if (isAlphaVisible(alpha)) this.sizeWithoutAir--; @@ -415,11 +424,11 @@ public class RenderDataPointReducingList } - long[] datas = this.data; + LongArrayList datas = this.data; int writeIndex = 0; for (int readIndex = this.getLowest(); readIndex != NULL; readIndex = this.getHigher(readIndex)) { - if (datas[readIndex] != DEFAUlT_DATA) + if (datas.getLong(readIndex) != DEFAUlT_DATA) { this.setSortingIndex(writeIndex++, readIndex); } @@ -434,8 +443,8 @@ public class RenderDataPointReducingList this.setBigger(smaller, bigger); } - this.smallest = this.sortingArray[0]; - this.biggest = this.sortingArray[writeIndex - 1]; + this.smallest = this.sortingArray.getShort(0); + this.biggest = this.sortingArray.getShort(writeIndex - 1); this.setSmaller(this.getSmallest(), NULL); this.setBigger(this.getBiggest(), NULL); } @@ -447,7 +456,7 @@ public class RenderDataPointReducingList @VisibleForTesting public void sortBySize(int size) { - short[] array = this.sortingArray; + ShortArrayList array = this.sortingArray; it.unimi.dsi.fastutil.Arrays.quickSort( 0, size, @@ -462,7 +471,7 @@ public class RenderDataPointReducingList // swapper (int index1, int index2) -> { - ShortArrays.swap(array, index1, index2); + ShortArrays.swap(array.elements(), index1, index2); } ); } @@ -474,7 +483,7 @@ public class RenderDataPointReducingList @VisibleForTesting public void sortByPosition(int size) { - short[] array = this.sortingArray; + ShortArrayList array = this.sortingArray; it.unimi.dsi.fastutil.Arrays.quickSort( 0, size, @@ -489,7 +498,7 @@ public class RenderDataPointReducingList // swapper (int index1, int index2) -> { - ShortArrays.swap(array, index1, index2); + ShortArrays.swap(array.elements(), index1, index2); } ); } @@ -913,15 +922,15 @@ public class RenderDataPointReducingList public int getSizeWithAir() { return Short.toUnsignedInt(this.sizeWithAir); } public int getSizeWithoutAir() { return Short.toUnsignedInt(this.sizeWithoutAir); } - public int getSortingIndex(int index) { return Short.toUnsignedInt(this.sortingArray[index]); } + public int getSortingIndex(int index) { return Short.toUnsignedInt(this.sortingArray.getShort(index)); } - public int getLower(int index) { return ((int) (this.links[index] >>> LOWER_SHIFT)) & LINK_MASK; } - public int getHigher(int index) { return ((int) (this.links[index] >>> HIGHER_SHIFT)) & LINK_MASK; } + public int getLower(int index) { return ((int) (this.links.getLong(index) >>> LOWER_SHIFT)) & LINK_MASK; } + public int getHigher(int index) { return ((int) (this.links.getLong(index) >>> HIGHER_SHIFT)) & LINK_MASK; } - public int getSmaller(int index) { return ((int) (this.links[index] >>> SMALLER_SHIFT)) & LINK_MASK; } - public int getBigger(int index) { return ((int) (this.links[index] >>> BIGGER_SHIFT)) & LINK_MASK; } + public int getSmaller(int index) { return ((int) (this.links.getLong(index) >>> SMALLER_SHIFT)) & LINK_MASK; } + public int getBigger(int index) { return ((int) (this.links.getLong(index) >>> BIGGER_SHIFT)) & LINK_MASK; } - public long getData(int index) { return this.data[index]; } + public long getData(int index) { return this.data.getLong(index); } public int getMinY(int index) { return RenderDataPointUtil.getYMin(this.getData(index)); } public int getMaxY(int index) { return RenderDataPointUtil.getYMax(this.getData(index)); } @@ -955,62 +964,62 @@ public class RenderDataPointReducingList public void setSizeWithAir(int sizeWithAir) { this.sizeWithAir = (short)(sizeWithAir); } public void setSizeWithoutAir(int sizeWithoutAir) { this.sizeWithoutAir = (short)(sizeWithoutAir); } - public void setSortingIndex(int index, int to) { this.sortingArray[index] = (short)(to); } + public void setSortingIndex(int index, int to) { this.sortingArray.set(index, (short)to); } public void setLower(int index, int lowerIndex) { - this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << LOWER_SHIFT)) | (((long)(lowerIndex & LINK_MASK)) << LOWER_SHIFT); + this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << LOWER_SHIFT)) | (((long)(lowerIndex & LINK_MASK)) << LOWER_SHIFT)); } public void setHigher(int index, int higherIndex) { - this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << HIGHER_SHIFT)) | (((long)(higherIndex & LINK_MASK)) << HIGHER_SHIFT); + this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << HIGHER_SHIFT)) | (((long)(higherIndex & LINK_MASK)) << HIGHER_SHIFT)); } public void setSmaller(int index, int smallerIndex) { - this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << SMALLER_SHIFT)) | (((long)(smallerIndex & LINK_MASK)) << SMALLER_SHIFT); + this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << SMALLER_SHIFT)) | (((long)(smallerIndex & LINK_MASK)) << SMALLER_SHIFT)); } public void setBigger(int index, int biggerIndex) { - this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << BIGGER_SHIFT)) | (((long)(biggerIndex & LINK_MASK)) << BIGGER_SHIFT); + this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << BIGGER_SHIFT)) | (((long)(biggerIndex & LINK_MASK)) << BIGGER_SHIFT)); } - public void setData(int index, long data) { this.data[index] = data; } + public void setData(int index, long data) { this.data.set(index, data); } public void setMinY(int index, int minY) { - this.data[index] = (this.data[index] & ~RenderDataPointUtil.DEPTH_SHIFTED_MASK) | ((minY & RenderDataPointUtil.DEPTH_MASK) << RenderDataPointUtil.DEPTH_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~RenderDataPointUtil.DEPTH_SHIFTED_MASK) | ((minY & RenderDataPointUtil.DEPTH_MASK) << RenderDataPointUtil.DEPTH_SHIFT)); } public void setMaxY(int index, int maxY) { - this.data[index] = (this.data[index] & ~RenderDataPointUtil.HEIGHT_SHIFTED_MASK) | ((maxY & RenderDataPointUtil.HEIGHT_MASK) << RenderDataPointUtil.HEIGHT_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~RenderDataPointUtil.HEIGHT_SHIFTED_MASK) | ((maxY & RenderDataPointUtil.HEIGHT_MASK) << RenderDataPointUtil.HEIGHT_SHIFT)); } public void setRed(int index, int red) { - this.data[index] = (this.data[index] & ~(RenderDataPointUtil.RED_MASK << RenderDataPointUtil.RED_SHIFT)) | ((red & RenderDataPointUtil.RED_MASK) << RenderDataPointUtil.RED_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.RED_MASK << RenderDataPointUtil.RED_SHIFT)) | ((red & RenderDataPointUtil.RED_MASK) << RenderDataPointUtil.RED_SHIFT)); } public void setGreen(int index, int green) { - this.data[index] = (this.data[index] & ~(RenderDataPointUtil.GREEN_MASK << RenderDataPointUtil.GREEN_SHIFT)) | ((green & RenderDataPointUtil.GREEN_MASK) << RenderDataPointUtil.GREEN_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.GREEN_MASK << RenderDataPointUtil.GREEN_SHIFT)) | ((green & RenderDataPointUtil.GREEN_MASK) << RenderDataPointUtil.GREEN_SHIFT)); } public void setBlue(int index, int blue) { - this.data[index] = (this.data[index] & ~(RenderDataPointUtil.BLUE_MASK << RenderDataPointUtil.BLUE_SHIFT)) | ((blue & RenderDataPointUtil.BLUE_MASK) << RenderDataPointUtil.BLUE_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.BLUE_MASK << RenderDataPointUtil.BLUE_SHIFT)) | ((blue & RenderDataPointUtil.BLUE_MASK) << RenderDataPointUtil.BLUE_SHIFT)); } public void setAlpha(int index, int alpha) { alpha >>>= RenderDataPointUtil.ALPHA_DOWNSIZE_SHIFT; - this.data[index] = (this.data[index] & ~(RenderDataPointUtil.ALPHA_MASK << RenderDataPointUtil.ALPHA_SHIFT)) | ((alpha & RenderDataPointUtil.ALPHA_MASK) << RenderDataPointUtil.ALPHA_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.ALPHA_MASK << RenderDataPointUtil.ALPHA_SHIFT)) | ((alpha & RenderDataPointUtil.ALPHA_MASK) << RenderDataPointUtil.ALPHA_SHIFT)); } public void setBlockLight(int index, int blockLight) { - this.data[index] = (this.data[index] & ~(RenderDataPointUtil.BLOCK_LIGHT_MASK << RenderDataPointUtil.BLOCK_LIGHT_SHIFT)) | ((blockLight & RenderDataPointUtil.BLOCK_LIGHT_MASK) << RenderDataPointUtil.BLOCK_LIGHT_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.BLOCK_LIGHT_MASK << RenderDataPointUtil.BLOCK_LIGHT_SHIFT)) | ((blockLight & RenderDataPointUtil.BLOCK_LIGHT_MASK) << RenderDataPointUtil.BLOCK_LIGHT_SHIFT)); } public void setSkyLight(int index, int skyLight) { - this.data[index] = (this.data[index] & ~(RenderDataPointUtil.SKY_LIGHT_MASK << RenderDataPointUtil.SKY_LIGHT_SHIFT)) | ((skyLight & RenderDataPointUtil.SKY_LIGHT_MASK) << RenderDataPointUtil.SKY_LIGHT_SHIFT); + this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.SKY_LIGHT_MASK << RenderDataPointUtil.SKY_LIGHT_SHIFT)) | ((skyLight & RenderDataPointUtil.SKY_LIGHT_MASK) << RenderDataPointUtil.SKY_LIGHT_SHIFT)); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index fb8faa06c..abf7d1cce 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -25,6 +25,8 @@ import com.seibel.distanthorizons.core.logging.SpamReducedLogger; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView; import com.seibel.distanthorizons.coreapi.ModInfo; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** @@ -67,6 +69,8 @@ public class RenderDataPointUtil public static final boolean RUN_VALIDATION = ModInfo.IS_DEV_BUILD; + private static final Logger LOGGER = LogManager.getLogger(); + public final static int EMPTY_DATA = 0; public final static int MAX_WORLD_Y_SIZE = 4096; @@ -298,9 +302,11 @@ public class RenderDataPointUtil } else { - RenderDataPointReducingList list = new RenderDataPointReducingList(sourceData); - list.reduce(output.verticalSize()); - list.copyTo(output); + try (RenderDataPointReducingList list = new RenderDataPointReducingList(sourceData)) + { + list.reduce(output.verticalSize()); + list.copyTo(output); + } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java index f04e2b283..9ae9b9fc2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java @@ -26,6 +26,7 @@ import com.seibel.distanthorizons.coreapi.ModInfo; import net.jpountz.lz4.LZ4FrameInputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.tukaani.xz.ResettableArrayCache; import org.tukaani.xz.XZInputStream; import java.io.*; @@ -42,6 +43,8 @@ import java.io.*; */ public class DhDataInputStream extends DataInputStream { + private static final ThreadLocal LZMA_RESETTABLE_ARRAY_CACHE_GETTER = ThreadLocal.withInitial(() -> new ResettableArrayCache(new LzmaArrayCache())); + private static final Logger LOGGER = LogManager.getLogger(); @@ -60,8 +63,12 @@ public class DhDataInputStream extends DataInputStream case LZ4: return new LZ4FrameInputStream(stream); case LZMA2: + // using an array cache significantly reduces GC pressure + ResettableArrayCache arrayCache = LZMA_RESETTABLE_ARRAY_CACHE_GETTER.get(); + arrayCache.reset(); + // Note: all LZMA/XZ compressors can be decompressed using this same InputStream - return new XZInputStream(stream); + return new XZInputStream(stream, arrayCache); default: throw new IllegalArgumentException("No compressor defined for [" + compressionMode + "]"); @@ -75,6 +82,23 @@ public class DhDataInputStream extends DataInputStream } } + + @Override + public int read() throws IOException + { + try + { + return super.read(); + } + catch (EOFException ignore) + { + // there's a bug with XZInputStream + // where it throws an EOFException instead + // of returning -1 as defined by DataInputStream.read() + return -1; + } + } + @Override public void close() throws IOException { /* Do nothing. */ } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java index 5c982130c..ecafc025f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java @@ -53,7 +53,7 @@ public class LzmaArrayCache extends ArrayCache public byte[] getByteArray(int size, boolean fillWithZeros) { ArrayList cacheList = this.byteCache.computeIfAbsent(size, (newSize) -> new ArrayList<>(4)); - if (cacheList.size() == 0) + if (cacheList.isEmpty()) { return new byte[size]; } diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyDto.java b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java index 745014dec..fb1c5603f 100644 --- a/core/src/test/java/testItems/sql/TestCompoundKeyDto.java +++ b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java @@ -61,4 +61,10 @@ public class TestCompoundKeyDto implements IBaseDTO return this.id + ", " + this.value; } + @Override + public void close() + { /* no closing needed */ } + + + } diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java index c76e65c65..509af940d 100644 --- a/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java +++ b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java @@ -20,10 +20,13 @@ package testItems.sql; import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; +import org.jetbrains.annotations.Nullable; import java.io.File; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -45,22 +48,33 @@ public class TestCompoundKeyRepo extends AbstractDhRepo objectMap) throws ClassCastException + @Override + protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, DhChunkPos pos) throws SQLException { - int xPos = (int) objectMap.get("XPos"); - int zPos = (int) objectMap.get("ZPos"); - String value = (String) objectMap.get("Value"); + statement.setInt(index++, pos.getX()); + statement.setInt(index++, pos.getZ()); + return index; + } + + + + @Override + @Nullable + public TestCompoundKeyDto convertResultSetToDto(ResultSet result) throws ClassCastException, SQLException + { + int xPos = result.getInt("XPos"); + int zPos = result.getInt("ZPos"); + String value = result.getString("Value"); return new TestCompoundKeyDto(new DhChunkPos(xPos, zPos), value); } diff --git a/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java index 9df6a16c8..edcd6d4c3 100644 --- a/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java +++ b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java @@ -19,10 +19,13 @@ package testItems.sql; +import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; +import org.jetbrains.annotations.Nullable; import java.io.File; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -43,24 +46,33 @@ public class TestPrimaryKeyRepo extends AbstractDhRepo objectMap) throws ClassCastException + @Override + protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, Integer id) throws SQLException { - int id = (int) objectMap.get("Id"); - String value = (String) objectMap.get("Value"); - long longValue = (Long) objectMap.get("LongValue"); - byte byteValue = (Byte) objectMap.get("ByteValue"); + statement.setInt(index++, id); + return index; + } + + + @Override + @Nullable + public TestSingleKeyDto convertResultSetToDto(ResultSet result) throws ClassCastException, SQLException + { + int id = result.getInt("Id"); + String value = result.getString("Value"); + long longValue = result.getLong("LongValue"); + byte byteValue = result.getByte("ByteValue"); return new TestSingleKeyDto(id, value, longValue, byteValue); } diff --git a/core/src/test/java/testItems/sql/TestSingleKeyDto.java b/core/src/test/java/testItems/sql/TestSingleKeyDto.java index 90709a162..1f7feff79 100644 --- a/core/src/test/java/testItems/sql/TestSingleKeyDto.java +++ b/core/src/test/java/testItems/sql/TestSingleKeyDto.java @@ -67,4 +67,10 @@ public class TestSingleKeyDto implements IBaseDTO return this.id + ", " + this.value + ", " + this.longValue + ", " + this.byteValue; } + @Override + public void close() + { /* no closing needed */ } + + + } diff --git a/core/src/test/java/tests/DhFullDataSourceCacheTest.java b/core/src/test/java/tests/DhFullDataSourceCacheTest.java new file mode 100644 index 000000000..da2ee7d8d --- /dev/null +++ b/core/src/test/java/tests/DhFullDataSourceCacheTest.java @@ -0,0 +1,220 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.DatabaseUpdater; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import testItems.sql.TestCompoundKeyDto; +import testItems.sql.TestCompoundKeyRepo; +import testItems.sql.TestPrimaryKeyRepo; +import testItems.sql.TestSingleKeyDto; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Map; + +/** + * Validates {@link AbstractDhRepo} is set up correctly. + */ +public class DhFullDataSourceCacheTest +{ + public static String DATABASE_TYPE = "jdbc:sqlite"; + public static String DB_FILE_NAME = "test.sqlite"; + + + + @BeforeClass + public static void testSetup() + { + File dbFile = new File(DB_FILE_NAME); + if (dbFile.exists()) + { + Assert.assertTrue("unable to delete old test DB File.", dbFile.delete()); + } + } + + + + @Test + public void test() + { + FullDataSourceV2Repo repo = null; + try + { + repo = new FullDataSourceV2Repo(DATABASE_TYPE, new File(DB_FILE_NAME)); + + + + //========================// + // create test datasource // + //========================// + + long pos = DhSectionPos.encode((byte)6, 1, 2); + FullDataPointIdMap dataMapping = new FullDataPointIdMap(pos); + LongArrayList[] fullDataArray = new LongArrayList[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + + for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++) + { + fullDataArray[i] = new LongArrayList(1); + + for (int j = 0; j < 32; j++) + { + fullDataArray[i].add(FullDataPointUtil.encode(j, 1, j, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT)); + } + } + + byte[] columnGenStep = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + Arrays.fill(columnGenStep, (byte)3); + + byte[] columnWorldCompressionMode = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + Arrays.fill(columnWorldCompressionMode, (byte)3); + + + + FullDataSourceV2 originalDataSource = FullDataSourceV2.createWithData(pos, dataMapping, fullDataArray, columnGenStep, columnWorldCompressionMode); + FullDataSourceV2DTO originalDto = FullDataSourceV2DTO.CreateFromDataSource(originalDataSource, EDhApiDataCompressionMode.LZMA2); + repo.save(originalDto); + + + + //=========================// + // assert DTO data is the same // + //=========================// + + FullDataSourceV2DTO savedDto = repo.getByKey(pos); + + Assert.assertNotNull("Failed to find DTO", savedDto); + Assert.assertEquals("Pos mismatch", originalDto.pos, savedDto.pos); + assertArraysAreEqual(originalDto.compressedDataByteArray, savedDto.compressedDataByteArray); + assertArraysAreEqual(originalDto.compressedColumnGenStepByteArray, savedDto.compressedColumnGenStepByteArray); + assertArraysAreEqual(originalDto.compressedWorldCompressionModeByteArray, savedDto.compressedWorldCompressionModeByteArray); + + + + //====================================// + // assert dataSource data is the same // + //====================================// + + FullDataSourceV2 savedDataSource = savedDto.createUnitTestDataSource(); + + Assert.assertNotNull("Failed to create DataSource", savedDataSource); + Assert.assertEquals("Pos mismatch", originalDataSource.getPos(), savedDataSource.getPos()); + assertArraysAreEqual(originalDataSource.columnGenerationSteps, savedDataSource.columnGenerationSteps); + assertArraysAreEqual(originalDataSource.columnWorldCompressionMode, savedDataSource.columnWorldCompressionMode); + Assert.assertTrue(originalDataSource.dataPoints.length == savedDataSource.dataPoints.length); + for (int i = 0; i < FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH; i++) + { + assertArraysAreEqual(originalDataSource.dataPoints[i], savedDataSource.dataPoints[i]); + } + + + + + //=========================// + // (optional) loop forever // + //=========================// + + // this can be set to "true" + // so we can profile the DTO creation process and + // see if there are any object leaks and how the GC + // handles it + if (false) + { + System.out.println("Initial save/get success, starting long update test for GC validation..."); + + for (int i = 0; i < 1_000_000; i++) + { + repo.save(originalDto); + try (FullDataSourceV2DTO pooledDto = repo.getByKey(pos)) + { + Assert.assertNotNull(savedDto); + Assert.assertEquals(originalDto.pos, savedDto.pos); + assertArraysAreEqual(originalDto.compressedDataByteArray, savedDto.compressedDataByteArray); + assertArraysAreEqual(originalDto.compressedColumnGenStepByteArray, savedDto.compressedColumnGenStepByteArray); + assertArraysAreEqual(originalDto.compressedWorldCompressionModeByteArray, savedDto.compressedWorldCompressionModeByteArray); + } + } + } + } + catch (Exception e) + { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + finally + { + if (repo != null) + { + repo.close(); + } + } + } + + + //================// + // helper methods // + //================// + + private static void assertArraysAreEqual(ByteArrayList expectedArray, ByteArrayList actualArray) + { + Assert.assertEquals("size mismatch", expectedArray.size(), actualArray.size()); + + for (int i = 0; i < expectedArray.size(); i++) + { + byte expectedNumb = expectedArray.getByte(i); + byte actualNumb = actualArray.getByte(i); + + Assert.assertEquals("value mismatch at index ["+i+"]", expectedNumb, actualNumb); + } + } + + private static void assertArraysAreEqual(LongArrayList expectedArray, LongArrayList actualArray) + { + Assert.assertEquals("size mismatch", expectedArray.size(), actualArray.size()); + + for (int i = 0; i < expectedArray.size(); i++) + { + long expectedNumb = expectedArray.getLong(i); + long actualNumb = actualArray.getLong(i); + + Assert.assertEquals("value mismatch at index ["+i+"]", expectedNumb, actualNumb); + } + } + + +} diff --git a/core/src/test/java/tests/DhRepoSqliteTest.java b/core/src/test/java/tests/DhRepoSqliteTest.java index bb9707b6c..ebd467ecf 100644 --- a/core/src/test/java/tests/DhRepoSqliteTest.java +++ b/core/src/test/java/tests/DhRepoSqliteTest.java @@ -31,6 +31,7 @@ import testItems.sql.TestPrimaryKeyRepo; import testItems.sql.TestSingleKeyDto; import java.io.File; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -71,8 +72,10 @@ public class DhRepoSqliteTest //==========================// // check that the schema table is created - Map autoUpdateTablePresentResult = primaryKeyRepo.queryDictionaryFirst("SELECT name FROM sqlite_master WHERE type='table' AND name='"+DatabaseUpdater.SCHEMA_TABLE_NAME+"';"); - if (autoUpdateTablePresentResult == null || autoUpdateTablePresentResult.get("name") == null) + ResultSet autoUpdateTablePresentResult = primaryKeyRepo.query(primaryKeyRepo.createPreparedStatement("SELECT name FROM sqlite_master WHERE type='table' AND name='"+DatabaseUpdater.SCHEMA_TABLE_NAME+"';")); + if (autoUpdateTablePresentResult == null + || !autoUpdateTablePresentResult.next() + || autoUpdateTablePresentResult.getString("name") == null) { Assert.fail("Auto DB update table missing."); }