From f11e9a142f3b891aeee67c4c8be858f5ca25f1dd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 17:25:37 -0500 Subject: [PATCH] Add FullDataSourceV2 pooling and replace long[][] arrays with LongArrayList[] --- .../methods/data/DhApiTerrainDataRepo.java | 7 +- .../fullData/sources/FullDataSourceV1.java | 10 + .../fullData/sources/FullDataSourceV2.java | 230 ++++++++++-------- .../render/ColumnRenderSource.java | 7 +- .../FullDataToRenderDataTransformer.java | 15 +- .../transformers/LodDataBuilder.java | 11 +- .../file/AbstractNewDataSourceHandler.java | 27 +- .../core/file/IDataSource.java | 6 +- .../FullDataSourceProviderV2.java | 12 +- .../renderfile/RenderSourceFileHandler.java | 8 +- .../SubDimensionLevelMatcher.java | 13 +- .../core/sql/dto/FullDataSourceV2DTO.java | 62 +++-- 12 files changed, 248 insertions(+), 160 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 933b29210..d922064f4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -46,6 +46,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.coreapi.util.math.Vec3d; import com.seibel.distanthorizons.coreapi.util.math.Vec3f; import com.seibel.distanthorizons.coreapi.util.math.Vec3i; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -220,10 +221,10 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo { // attempt to get the LOD data from the data source FullDataPointIdMap mapping = dataSource.mapping; - long[] dataColumn = dataSource.get(relativePos.x, relativePos.z); + LongArrayList dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { - int dataColumnIndexCount = dataColumn.length; + int dataColumnIndexCount = dataColumn.size(); DhApiTerrainDataPoint[] returnArray = new DhApiTerrainDataPoint[dataColumnIndexCount]; long dataPoint; @@ -234,7 +235,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo // search for a datapoint that contains the block y position for (int i = 0; i < dataColumnIndexCount; i++) { - dataPoint = dataColumn[i]; + dataPoint = dataColumn.getLong(i); if (!getSpecificYCoordinate) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 22b9db0c7..0558dfe4f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -387,6 +387,16 @@ public class FullDataSourceV1 implements IDataSource + //==================// + // override methods // + //==================// + + @Override + public void close() throws Exception + { /* not currently needed */ } + + + //================// // helper classes // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 9994c39be..54d2fff2d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; @@ -51,7 +52,7 @@ import java.util.concurrent.locks.ReentrantLock; * @see FullDataPointUtilV2 * @see FullDataSourceV1 */ -public class FullDataSourceV2 implements IDataSource +public class FullDataSourceV2 implements IDataSource, AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ @@ -98,7 +99,7 @@ public class FullDataSourceV2 implements IDataSource * TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately * there's something in the render data logic that expects this order so we can't change it right now */ - public long[][] dataPoints; + public LongArrayList[] dataPoints; public boolean isEmpty; public boolean applyToParent = false; @@ -113,7 +114,7 @@ public class FullDataSourceV2 implements IDataSource private FullDataSourceV2(DhSectionPos pos) { this.pos = pos; - this.dataPoints = new long[WIDTH * WIDTH][]; + this.dataPoints = new LongArrayList[WIDTH * WIDTH]; this.mapping = new FullDataPointIdMap(pos); this.isEmpty = true; @@ -122,8 +123,8 @@ public class FullDataSourceV2 implements IDataSource this.columnGenerationSteps = new byte[WIDTH * WIDTH]; } - public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } - private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) + public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } + private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps) { LodUtil.assertTrue(data.length == WIDTH * WIDTH); @@ -150,7 +151,7 @@ public class FullDataSourceV2 implements IDataSource // Note: this logic only works if the data point data is the same between both versions byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; - long[][] dataPoints = new long[WIDTH * WIDTH][]; + LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH]; for (int x = 0; x < WIDTH; x++) { for (int z = 0; z < WIDTH; z++) @@ -159,7 +160,7 @@ public class FullDataSourceV2 implements IDataSource if (dataColumn != null && dataColumn.length != 0) { int index = relativePosToIndex(x, z); - dataPoints[index] = dataColumn; + dataPoints[index] = new LongArrayList(dataColumn); // convert the data point format @@ -212,13 +213,13 @@ public class FullDataSourceV2 implements IDataSource long[] legacyDataColumn = legacyData.get(x, z); if (legacyDataColumn != null && legacyDataColumn.length != 0) { - long[] newDataColumn = fullDataSource.get(x, z); + LongArrayList newDataColumn = fullDataSource.get(x, z); if (newDataColumn == null) { LodUtil.assertNotReach("Accessor column mismatch"); } - else if (legacyDataColumn.length != newDataColumn.length) + else if (legacyDataColumn.length != newDataColumn.size()) { LodUtil.assertNotReach("Accessor column length mismatch"); } @@ -226,7 +227,7 @@ public class FullDataSourceV2 implements IDataSource { for (int i = 0; i < legacyDataColumn.length; i++) { - if (legacyDataColumn[i] != newDataColumn[i]) + if (legacyDataColumn[i] != newDataColumn.getLong(i)) { LodUtil.assertNotReach("Data mismatch"); } @@ -247,7 +248,7 @@ public class FullDataSourceV2 implements IDataSource // data // //======// - public long[] get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } + public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Override public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } @@ -309,7 +310,7 @@ public class FullDataSourceV2 implements IDataSource { int index = relativePosToIndex(x, z); - long[] newDataArray = inputDataSource.dataPoints[index]; + LongArrayList newDataArray = inputDataSource.dataPoints[index]; if (newDataArray != null) { byte thisGenState = this.columnGenerationSteps[index]; @@ -318,11 +319,15 @@ public class FullDataSourceV2 implements IDataSource if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value && thisGenState <= inputGenState) { - long[] oldDataArray = this.dataPoints[index]; + LongArrayList oldDataArray = this.dataPoints[index]; // copy over the new data - this.dataPoints[index] = new long[newDataArray.length]; - System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); + if (this.dataPoints[index] == null) + { + this.dataPoints[index] = new LongArrayList(new long[newDataArray.size()]); + } + this.dataPoints[index].clear(); + this.dataPoints[index].addAll(newDataArray); this.remapDataColumn(index, remappedIds); if (RUN_DATA_ORDER_VALIDATION) @@ -383,8 +388,8 @@ public class FullDataSourceV2 implements IDataSource this.columnGenerationSteps[recipientIndex] = inputGenStep; // data points - long[] oldDataArray = this.dataPoints[recipientIndex]; - long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); + LongArrayList oldDataArray = this.dataPoints[recipientIndex]; + LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); this.dataPoints[recipientIndex] = mergedInputDataArray; if (RUN_DATA_ORDER_VALIDATION) @@ -427,9 +432,13 @@ public class FullDataSourceV2 implements IDataSource } return minWorldGenStepValue; } - private static long[] mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) + + private static ThreadLocal mergeTwoByTwoTempList = ThreadLocal.withInitial(() -> new LongArrayList(20)); + private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { - ArrayList newColumnList = new ArrayList<>(); + LongArrayList newColumnList = mergeTwoByTwoTempList.get(); + newColumnList.clear(); + // special numbers: // -2 = the column's height hasn't been determined yet @@ -466,8 +475,8 @@ public class FullDataSourceV2 implements IDataSource for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++) { // TODO throw an assertion if the column isn't in top-down order or just fix it... - long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; - if (inputDataArray == null || inputDataArray.length == 0) + LongArrayList inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; + if (inputDataArray == null || inputDataArray.size() == 0) { currentDatapointIndex[colIndex] = -1; continue; @@ -476,7 +485,7 @@ public class FullDataSourceV2 implements IDataSource // determine the last index (the lowest data point) for each column if (currentDatapointIndex[colIndex] == -2) { - currentDatapointIndex[colIndex] = inputDataArray.length - 1; + currentDatapointIndex[colIndex] = inputDataArray.size() - 1; if (RUN_DATA_ORDER_VALIDATION) { @@ -491,7 +500,7 @@ public class FullDataSourceV2 implements IDataSource // went over the end continue; } - long datapoint = inputDataArray[dataPointIndex]; + long datapoint = inputDataArray.getLong(dataPointIndex); int datapointMinY = FullDataPointUtilV2.getBottomY(datapoint); int numbOfBlocksTall = FullDataPointUtilV2.getHeight(datapoint); @@ -575,10 +584,10 @@ public class FullDataSourceV2 implements IDataSource // convert the arraylist to an array - long[]mergedInputDataArray = new long[newColumnList.size()]; - for (int i = 0; i < mergedInputDataArray.length; i++) + LongArrayList mergedInputDataArray = new LongArrayList(new long[newColumnList.size()]); + for (int i = 0; i < mergedInputDataArray.size(); i++) { - mergedInputDataArray[i] = newColumnList.get(i); + mergedInputDataArray.set(i, newColumnList.getLong(i)); } @@ -586,10 +595,10 @@ public class FullDataSourceV2 implements IDataSource // TODO why is this sometimes necessary? What did I (James) screw up that causes the mergedInputDataArray // to sometimes be in a different order? Is it potentially related to what detail level is coming in? { - long firstDataPoint = mergedInputDataArray[0]; + long firstDataPoint = mergedInputDataArray.getLong(0); int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = mergedInputDataArray[mergedInputDataArray.length - 1]; + long lastDataPoint = mergedInputDataArray.getLong(mergedInputDataArray.size() - 1); int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -597,11 +606,11 @@ public class FullDataSourceV2 implements IDataSource // reverse the array so index 0 is the highest, // this is necessary for later logic // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - for(int i = 0; i < mergedInputDataArray.length / 2; i++) + for(int i = 0; i < mergedInputDataArray.size() / 2; i++) { - long temp = mergedInputDataArray[i]; - mergedInputDataArray[i] = mergedInputDataArray[mergedInputDataArray.length - i - 1]; - mergedInputDataArray[mergedInputDataArray.length - i - 1] = temp; + long temp = mergedInputDataArray.getLong(i); + mergedInputDataArray.set(i, mergedInputDataArray.getLong(mergedInputDataArray.size() - i - 1)); + mergedInputDataArray.set(mergedInputDataArray.size() - i - 1, temp); } } } @@ -615,15 +624,15 @@ public class FullDataSourceV2 implements IDataSource */ private void remapDataColumn(int dataPointIndex, int[] remappedIds) { - long[] dataColumn = this.dataPoints[dataPointIndex]; - for (int i = 0; i < dataColumn.length; i++) + LongArrayList dataColumn = this.dataPoints[dataPointIndex]; + for (int i = 0; i < dataColumn.size(); i++) { - dataColumn[i] = FullDataPointUtilV2.remap(remappedIds, dataColumn[i]); + dataColumn.set(i, FullDataPointUtilV2.remap(remappedIds, dataColumn.getLong(i))); } } - private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray) + private static boolean areDataColumnsDifferent(LongArrayList oldDataArray, LongArrayList newDataArray) { - if (oldDataArray == null || oldDataArray.length != newDataArray.length) + if (oldDataArray == null || oldDataArray.size() != newDataArray.size()) { // new data was added/removed return true; @@ -631,8 +640,8 @@ public class FullDataSourceV2 implements IDataSource else { // check if the new column data is different - int oldArrayHash = Arrays.hashCode(oldDataArray); - int newArrayHash = Arrays.hashCode(newDataArray); + int oldArrayHash = oldDataArray.hashCode(); + int newArrayHash = newDataArray.hashCode(); return (newArrayHash != oldArrayHash); } } @@ -732,12 +741,12 @@ public class FullDataSourceV2 implements IDataSource * * @see FullDataSourceV2#dataPoints */ - public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, long[] dataArray) throws IllegalStateException + public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, LongArrayList dataArray) throws IllegalStateException { - long firstDataPoint = dataArray[0]; + long firstDataPoint = dataArray.getLong(0); int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = dataArray[dataArray.length - 1]; + long lastDataPoint = dataArray.getLong(dataArray.size() - 1); int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -767,7 +776,7 @@ public class FullDataSourceV2 implements IDataSource return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); } - public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) { int index = relativePosToIndex(relX, relZ); this.dataPoints[index] = longArray; @@ -778,9 +787,9 @@ public class FullDataSourceV2 implements IDataSource { // validate the incoming ID's int maxValidId = this.mapping.getMaxValidId(); - for (int i = 0; i < longArray.length; i++) + for (int i = 0; i < longArray.size(); i++) { - long dataPoint = longArray[i]; + long dataPoint = longArray.getLong(i); int id = FullDataPointUtilV2.getId(dataPoint); if (id > maxValidId) { @@ -858,64 +867,83 @@ public class FullDataSourceV2 implements IDataSource // pooling // //=========// - // TODO add pooled data sources - private static class Pooling + @Override + public void close() throws Exception { - /** used when pooling data sources */ - private final ArrayList cachedSources = new ArrayList<>(); - private final ReentrantLock cacheLock = new ReentrantLock(); - - - /** @return null if no pooled source exists */ - public FullDataSourceV1 tryGetPooledSource() - { - try - { - this.cacheLock.lock(); - - int index = this.cachedSources.size() - 1; - if (index == -1) - { - return null; - } - else - { - return this.cachedSources.remove(index); - } - } - finally - { - this.cacheLock.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called. - */ - public void returnPooledDataSource(FullDataSourceV1 dataSource) - { - if (dataSource == null) - { - return; - } - else if (this.cachedSources.size() > 25) - { - return; - } - - try - { - this.cacheLock.lock(); - this.cachedSources.add(dataSource); - } - finally - { - this.cacheLock.unlock(); - } - } - + returnPooledDataSource(this); } + /** used when pooling data sources */ + private static final ArrayList CACHED_SOURCES = new ArrayList<>(); + private static final ReentrantLock CACHE_LOCK = new ReentrantLock(); + + + /** @return an empty data source if non are cached */ + public static FullDataSourceV2 getPooledSource(DhSectionPos pos, boolean clearData) + { + try + { + CACHE_LOCK.lock(); + + int index = CACHED_SOURCES.size() - 1; + if (index == -1) + { + return createEmpty(pos); + } + else + { + FullDataSourceV2 dataSource = CACHED_SOURCES.remove(index);; + dataSource.pos = pos; + + if (clearData) + { + dataSource.mapping.clear(pos); + + for (int i = 0; i < dataSource.dataPoints.length; i++) + { + if (dataSource.dataPoints[i] != null) + { + dataSource.dataPoints[i].clear(); + } + } + + Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); + } + + return dataSource; + } + } + finally + { + CACHE_LOCK.unlock(); + } + } + + /** + * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. + * It just means a new source must be constructed next time {@link FullDataSourceV2#getPooledSource} is called. + */ + public static void returnPooledDataSource(FullDataSourceV2 dataSource) + { + if (dataSource == null) + { + return; + } + else if (CACHED_SOURCES.size() > 25) + { + return; + } + + try + { + CACHE_LOCK.lock(); + CACHED_SOURCES.add(dataSource); + } + finally + { + CACHE_LOCK.unlock(); + } + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 208aa4df5..fd2b8a011 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -37,6 +37,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.io.*; @@ -317,7 +318,7 @@ public class ColumnRenderSource implements IDataSource ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); int columnHash = columnArrayView.getDataHash(); - long[] dataColumn = inputFullDataSource.get(x, z); + LongArrayList dataColumn = inputFullDataSource.get(x, z); EDhApiWorldGenerationStep worldGenStep = inputFullDataSource.getWorldGenStepAtRelativePos(x, z); if (dataColumn != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) { @@ -476,6 +477,10 @@ public class ColumnRenderSource implements IDataSource return stringBuilder.toString(); } + @Override + public void close() throws Exception + { /* not currently needed */ } + //==============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index b4f12126c..28d7dd301 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -36,6 +36,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.util.HashSet; @@ -117,7 +118,7 @@ public class FullDataToRenderDataTransformer throwIfThreadInterrupted(); ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - long[] dataColumn = fullDataSource.get(x, z); + LongArrayList dataColumn = fullDataSource.get(x, z); convertColumnData(level, fullDataSource.mapping, baseX + x, baseZ + z, columnArrayView, dataColumn); } } @@ -159,7 +160,7 @@ public class FullDataToRenderDataTransformer private static void iterateAndConvert( IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, - ColumnArrayView renderColumnData, long[] fullColumnData) + ColumnArrayView renderColumnData, LongArrayList fullColumnData) { boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); @@ -171,9 +172,9 @@ public class FullDataToRenderDataTransformer int columnOffset = 0; // goes from the top down - for (int i = 0; i < fullColumnData.length; i++) + for (int i = 0; i < fullColumnData.size(); i++) { - long fullData = fullColumnData[i]; + long fullData = fullColumnData.getLong(i); int bottomY = FullDataPointUtilV2.getBottomY(fullData); int blockHeight = FullDataPointUtilV2.getHeight(fullData); int id = FullDataPointUtilV2.getId(fullData); @@ -270,14 +271,14 @@ public class FullDataToRenderDataTransformer } // TODO what does this mean? - public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, long[] fullDataColumn) + public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, LongArrayList fullDataColumn) { - if (fullDataColumn == null || fullDataColumn.length == 0) + if (fullDataColumn == null || fullDataColumn.size() == 0) { return; } - int dataTotalLength = fullDataColumn.length; + int dataTotalLength = fullDataColumn.size(); if (dataTotalLength > columnArrayView.verticalSize()) { ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 2a3346f03..bcf79b56b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -131,7 +131,7 @@ public class LodDataBuilder { for (int chunkZ = 0; chunkZ < LodUtil.CHUNK_WIDTH; chunkZ++) { - LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); + LongArrayList longs = new LongArrayList(new long[chunkWrapper.getHeight() / 4]); int lastY = chunkWrapper.getMaxBuildHeight(); IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); IBlockStateWrapper blockState = AIR; @@ -201,7 +201,7 @@ public class LodDataBuilder } longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - dataSource.setSingleColumn(longs.toLongArray(), + dataSource.setSingleColumn(longs, chunkX + chunkOffsetX, chunkZ + chunkOffsetZ, EDhApiWorldGenerationStep.LIGHT); @@ -230,7 +230,7 @@ public class LodDataBuilder // AND the below loop won't run. int size = (columnDataPoints != null) ? columnDataPoints.size() : 0; - long[] packedDataPoints = new long[size]; + LongArrayList packedDataPoints = new LongArrayList(new long[size]); for (int index = 0; index < size; index++) { DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index); @@ -240,13 +240,14 @@ public class LodDataBuilder (IBlockStateWrapper) (dataPoint.blockStateWrapper) ); - packedDataPoints[index] = FullDataPointUtilV2.encode( + packedDataPoints.set(index, + FullDataPointUtilV2.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, (byte) (dataPoint.blockLightLevel), (byte) (dataPoint.skyLightLevel) - ); + )); } accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 3d804d013..f0489188c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -221,22 +221,23 @@ public abstract class AbstractNewDataSourceHandler } - // get or create the data source - TDataSource recipientDataSource = this.get(updatePos); - boolean dataModified = recipientDataSource.update(inputData, this.level); - - if (dataModified) + // try block allows for disposing of pooled data sources after the update is complete + try (TDataSource recipientDataSource = this.get(updatePos)) { - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(recipientDataSource); - this.repo.save(dto); - - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + boolean dataModified = recipientDataSource.update(inputData, this.level); + if (dataModified) { - if (listener != null) + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(recipientDataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) { - listener.OnDataSourceUpdated(recipientDataSource); + if (listener != null) + { + listener.OnDataSourceUpdated(recipientDataSource); + } } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 11738fe7a..0da2c34b6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -11,11 +11,13 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import java.io.IOException; /** - * Base for all data sources. + * Base for all data sources.

+ * + * AutoCloseable Can be implemented to allow for disposing of pooled data sources.

* * @param there are times when we need specifically a client level vs a more generic level */ -public interface IDataSource extends IBaseDTO +public interface IDataSource extends IBaseDTO, AutoCloseable { DhSectionPos getSectionPos(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 70dd8bf1e..0e1b9f34d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -162,11 +162,11 @@ public class FullDataSourceProviderV2 { // TODO maybe just set children update flags to true? // TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway - return FullDataSourceV2.createEmpty(pos); + return FullDataSourceV2.getPooledSource(pos, true); } @Override - protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.createEmpty(pos); } + protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.getPooledSource(pos, true); } @@ -248,9 +248,11 @@ public class FullDataSourceProviderV2 childReadLock.lock(); this.lockedPosSet.add(childPos); - FullDataSourceV2 dataSource = this.get(childPos); - this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); - this.repo.setApplyToParent(childPos, false); + try (FullDataSourceV2 dataSource = this.get(childPos)) + { + this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); + this.repo.setApplyToParent(childPos, false); + } } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 46f971f16..003e67270 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -92,8 +92,12 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler { + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + public DhSectionPos pos; public int levelMinY; @@ -113,7 +120,10 @@ public class FullDataSourceV2DTO implements IBaseDTO //========================// public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException - { return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), levelWrapper); } + { + FullDataSourceV2 dataSource = FullDataSourceV2.getPooledSource(this.pos, false); + return this.populateDataSource(dataSource, levelWrapper); + } public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } @@ -133,7 +143,7 @@ public class FullDataSourceV2DTO implements IBaseDTO } dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); - dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); + dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, this.compressionModeEnum); dataSource.mapping.clear(dataSource.getSectionPos()); // should only be null when used in a unit test @@ -163,7 +173,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // (de)serializing // //=================// - private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static CheckedByteArray writeDataSourceDataArrayToBlob(LongArrayList[] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -179,10 +189,10 @@ public class FullDataSourceV2DTO implements IBaseDTO int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; for (int xz = 0; xz < dataArrayLength; xz++) { - long[] dataColumn = dataArray[xz]; + LongArrayList dataColumn = dataArray[xz]; // write column length - short columnLength = (dataColumn != null) ? (short) dataColumn.length : 0; + short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; // a short is used instead of an int because at most we store 4096 vertical slices and a // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) compressedOut.writeShort(columnLength); @@ -190,7 +200,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // write column data (will be skipped if no data was present) for (int y = 0; y < columnLength; y++) { - compressedOut.writeLong(dataColumn[y]); + compressedOut.writeLong(dataColumn.getLong(y)); } } @@ -202,33 +212,55 @@ public class FullDataSourceV2DTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static long[][] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, LongArrayList[] existingDataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); + int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; + if (existingDataArray == null + || existingDataArray.length != dataArrayLength) + { + existingDataArray = new LongArrayList[dataArrayLength]; + } + + // read the data - int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; - long[][] dataArray = new long[dataArrayLength][]; - for (int xz = 0; xz < dataArray.length; xz++) + for (int xz = 0; xz < existingDataArray.length; xz++) { // read the column length - short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later - long[] dataColumn = new long[dataColumnLength]; + short dataColumnLength = compressedIn.readShort(); + + + // use the existing array if possible + LongArrayList dataColumn = existingDataArray[xz]; + if (dataColumn == null) + { + dataColumn = new LongArrayList(new long[dataColumnLength]); + existingDataArray[xz] = dataColumn; + } + + dataColumn.clear(); + dataColumn.ensureCapacity(dataColumnLength); + while (dataColumn.size() < dataColumnLength) + { + dataColumn.add(0); + } + // read column data (will be skipped if no data was present) for (int y = 0; y < dataColumnLength; y++) { long dataPoint = compressedIn.readLong(); - dataColumn[y] = dataPoint; + dataColumn.set(y, dataPoint); } - dataArray[xz] = dataColumn; + existingDataArray[xz] = dataColumn; } - return dataArray; + return existingDataArray; }