diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java index 2d9534787..d8abc9eb0 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.api.enums.config; /** - * UNCOMPRESSED
+ * MERGE_SAME_BLOCKS
* VISUALLY_EQUAL

* * @version 2024-3-27 @@ -32,8 +32,11 @@ public enum EDhApiWorldCompressionMode // when adding items up the API minor version // when removing items up the API major version - /** Every block/biome change is recorded in the database. */ - UNCOMPRESSED(0), + /** + * Every block/biome change is recorded in the database.
+ * This is what DH 2.0 and 2.0.1 all used by default and will store a lot of data. + */ + MERGE_SAME_BLOCKS(0), /** * Only visible block/biome changes are recorded in the database.
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 0d4a35e35..75ab0ec71 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; +import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; @@ -93,6 +94,11 @@ public class FullDataSourceV2 implements IDataSource * @see EDhApiWorldGenerationStep */ public byte[] columnGenerationSteps; + /** + * stores what world compression was used for each column. + * @see EDhApiWorldCompressionMode + */ + public byte[] columnWorldCompressionMode; /** * stored x/z, y
@@ -122,10 +128,11 @@ public class FullDataSourceV2 implements IDataSource // doesn't need to be populated since nothing has been generated yet // the default value of all 0's is adequate this.columnGenerationSteps = new byte[WIDTH * WIDTH]; + this.columnWorldCompressionMode = new byte[WIDTH * WIDTH]; } - public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } - private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps) + public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep, byte[] columnWorldCompressionMode) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep, columnWorldCompressionMode); } + private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps, byte[] columnWorldCompressionMode) { LodUtil.assertTrue(data.length == WIDTH * WIDTH); @@ -135,6 +142,7 @@ public class FullDataSourceV2 implements IDataSource this.isEmpty = false; this.columnGenerationSteps = columnGenerationSteps; + this.columnWorldCompressionMode = columnWorldCompressionMode; } public static FullDataSourceV2 createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } @@ -152,6 +160,7 @@ public class FullDataSourceV2 implements IDataSource // Note: this logic only works if the data point data is the same between both versions byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; + byte[] columnWorldCompressionMode = new byte[WIDTH * WIDTH]; LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH]; for (int x = 0; x < WIDTH; x++) { @@ -197,11 +206,12 @@ public class FullDataSourceV2 implements IDataSource // the old data sources didn't have a generation step written down // if the column has any data points, assume it's fully generated, otherwise assume it's empty columnGenerationSteps[index] = (columnHasNonAirBlock ? EDhApiWorldGenerationStep.LIGHT.value : EDhApiWorldGenerationStep.EMPTY.value); + columnWorldCompressionMode[index] = EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS.value; } } } - FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps); + FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps, columnWorldCompressionMode); // should only be used if debugging, this is a very expensive operation @@ -361,7 +371,10 @@ public class FullDataSourceV2 implements IDataSource } } + this.columnGenerationSteps[index] = inputGenState; + // always overwrite the compression mode since we're replacing this column + this.columnWorldCompressionMode[index] = inputDataSource.columnWorldCompressionMode[index]; this.isEmpty = false; } } @@ -407,6 +420,11 @@ public class FullDataSourceV2 implements IDataSource this.columnGenerationSteps[recipientIndex] = inputGenStep; + // world compression // + byte worldCompressionMode = determineHighestWorldCompressionForTwoByTwoColumn(inputDataSource.columnWorldCompressionMode, x, z); + this.columnWorldCompressionMode[recipientIndex] = worldCompressionMode; + + // data points // LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); @@ -465,6 +483,7 @@ public class FullDataSourceV2 implements IDataSource */ private static byte determineMinWorldGenStepForTwoByTwoColumn(byte[] columnGenerationSteps, int relX, int relZ) { + // TODO merge similar logic with determineHighestWorldCompressionForTwoByTwoColumn byte minWorldGenStepValue = Byte.MAX_VALUE; for (int x = 0; x < 2; x++) { @@ -477,6 +496,25 @@ public class FullDataSourceV2 implements IDataSource } return minWorldGenStepValue; } + /** + * The minimum value is used because we don't want to accidentally record that + * something was generated when it wasn't. + */ + private static byte determineHighestWorldCompressionForTwoByTwoColumn(byte[] columnCompressionMode, int relX, int relZ) + { + // TODO merge similar logic with determineMinWorldGenStepForTwoByTwoColumn + byte minWorldGenStepValue = Byte.MIN_VALUE; + for (int x = 0; x < 2; x++) + { + for (int z = 0; z < 2; z++) + { + int index = relativePosToIndex(x + relX, z + relZ); + byte worldGenStepValue = columnCompressionMode[index]; + minWorldGenStepValue = (byte) Math.max(minWorldGenStepValue, worldGenStepValue); + } + } + return minWorldGenStepValue; + } private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { LongArrayList newColumnList = new LongArrayList(); @@ -817,11 +855,12 @@ public class FullDataSourceV2 implements IDataSource return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); } - public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep, EDhApiWorldCompressionMode worldCompressionMode) { int index = relativePosToIndex(relX, relZ); this.dataPoints[index] = longArray; this.columnGenerationSteps[index] = worldGenStep.value; + this.columnWorldCompressionMode[index] = worldCompressionMode.value; if (RUN_UPDATE_DEV_VALIDATION) @@ -863,6 +902,7 @@ public class FullDataSourceV2 implements IDataSource int result = this.pos.hashCode(); result = 31 * result + Arrays.deepHashCode(this.dataPoints); result = 17 * result + Arrays.hashCode(this.columnGenerationSteps); + result = 43 * result + Arrays.hashCode(this.columnWorldCompressionMode); this.cachedHashCode = result; } @@ -938,6 +978,7 @@ public class FullDataSourceV2 implements IDataSource } Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); + Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0); } return dataSource; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index ddae6c4d1..35ec4a892 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.sql.dto; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; @@ -50,6 +51,8 @@ public class FullDataSourceV2DTO implements IBaseDTO /** @see EDhApiWorldGenerationStep */ public byte[] compressedColumnGenStepByteArray; + /** @see EDhApiWorldCompressionMode */ + public byte[] compressedWorldCompressionModeByteArray; public byte[] compressedMappingByteArray; @@ -71,11 +74,12 @@ public class FullDataSourceV2DTO implements IBaseDTO { CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum); byte[] compressedWorldGenStepByteArray = writeGenerationStepsToBlob(dataSource.columnGenerationSteps, compressionModeEnum); + byte[] compressedWorldCompressionModeByteArray = writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, compressionModeEnum); byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum); return new FullDataSourceV2DTO( dataSource.getSectionPos(), - checkedDataPointArray.checksum, compressedWorldGenStepByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, + checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, dataSource.levelMinY @@ -84,7 +88,7 @@ public class FullDataSourceV2DTO implements IBaseDTO public FullDataSourceV2DTO( DhSectionPos pos, - int dataChecksum, byte[] compressedColumnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray, + int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray, long lastModifiedUnixDateTime, long createdUnixDateTime, byte[] compressedMappingByteArray, boolean applyToParent, int levelMinY) @@ -92,6 +96,7 @@ public class FullDataSourceV2DTO implements IBaseDTO this.pos = pos; this.dataChecksum = dataChecksum; this.compressedColumnGenStepByteArray = compressedColumnGenStepByteArray; + this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray; this.dataFormatVersion = dataFormatVersion; this.compressionModeEnum = compressionModeEnum; @@ -133,10 +138,11 @@ public class FullDataSourceV2DTO implements IBaseDTO { if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) { - throw new IllegalStateException("There should only be one data format right now anyway."); + throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"]."); } dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); + dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, this.compressionModeEnum); dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); dataSource.mapping.clear(dataSource.getSectionPos()); @@ -260,6 +266,30 @@ public class FullDataSourceV2DTO implements IBaseDTO } + private static byte[] writeWorldCompressionModeToBlob(byte[] worldCompressionModeByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum); + + compressedOut.write(worldCompressionModeByteArray); + + compressedOut.flush(); + byteArrayOutputStream.close(); + + return byteArrayOutputStream.toByteArray(); + } + private static byte[] readBlobToWorldCompressionMode(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); + + byte[] worldCompressionModeByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + compressedIn.readFully(worldCompressionModeByteArray); + + return worldCompressionModeByteArray; + } + + private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index b325aacd7..619581a14 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -86,6 +86,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo