wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ColumnWorldCompressionMode to the database

Browse Source
This commit is contained in:
James Seibel
2024-03-28 07:36:02 -05:00
parent e83864fd02
commit d41af88494
6 changed files with 99 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java
+6 -3
View File
@@ -20,7 +20,7 @@
package com.seibel.distanthorizons.api.enums.config;
/**
* UNCOMPRESSED <br>
* MERGE_SAME_BLOCKS <br>
* VISUALLY_EQUAL <br><br>
*
* @version 2024-3-27
@@ -32,8 +32,11 @@ public enum EDhApiWorldCompressionMode
// when adding items up the API minor version
// when removing items up the API major version
/** Every block/biome change is recorded in the database. */
UNCOMPRESSED(0),
/**
* Every block/biome change is recorded in the database. <br>
* This is what DH 2.0 and 2.0.1 all used by default and will store a lot of data.
*/
MERGE_SAME_BLOCKS(0),
/**
* Only visible block/biome changes are recorded in the database. <Br>
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+45 -4
View File
@@ -19,6 +19,7 @@
package com.seibel.distanthorizons.core.dataObjects.fullData.sources;
import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder;
@@ -93,6 +94,11 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
* @see EDhApiWorldGenerationStep
*/
public byte[] columnGenerationSteps;
/**
* stores what world compression was used for each column.
* @see EDhApiWorldCompressionMode
*/
public byte[] columnWorldCompressionMode;
/**
* stored x/z, y <br>
@@ -122,10 +128,11 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// doesn't need to be populated since nothing has been generated yet
// the default value of all 0's is adequate
this.columnGenerationSteps = new byte[WIDTH * WIDTH];
this.columnWorldCompressionMode = new byte[WIDTH * WIDTH];
}
public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); }
private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps)
public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep, byte[] columnWorldCompressionMode) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep, columnWorldCompressionMode); }
private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps, byte[] columnWorldCompressionMode)
{
LodUtil.assertTrue(data.length == WIDTH * WIDTH);
@@ -135,6 +142,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
this.isEmpty = false;
this.columnGenerationSteps = columnGenerationSteps;
this.columnWorldCompressionMode = columnWorldCompressionMode;
}
public static FullDataSourceV2 createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); }
@@ -152,6 +160,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// Note: this logic only works if the data point data is the same between both versions
byte[] columnGenerationSteps = new byte[WIDTH * WIDTH];
byte[] columnWorldCompressionMode = new byte[WIDTH * WIDTH];
LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH];
for (int x = 0; x < WIDTH; x++)
{
@@ -197,11 +206,12 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// the old data sources didn't have a generation step written down
// if the column has any data points, assume it's fully generated, otherwise assume it's empty
columnGenerationSteps[index] = (columnHasNonAirBlock ? EDhApiWorldGenerationStep.LIGHT.value : EDhApiWorldGenerationStep.EMPTY.value);
columnWorldCompressionMode[index] = EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS.value;
}
}
}
FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps);
FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps, columnWorldCompressionMode);
// should only be used if debugging, this is a very expensive operation
@@ -361,7 +371,10 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
}
}
this.columnGenerationSteps[index] = inputGenState;
// always overwrite the compression mode since we're replacing this column
this.columnWorldCompressionMode[index] = inputDataSource.columnWorldCompressionMode[index];
this.isEmpty = false;
}
}
@@ -407,6 +420,11 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
this.columnGenerationSteps[recipientIndex] = inputGenStep;
// world compression //
byte worldCompressionMode = determineHighestWorldCompressionForTwoByTwoColumn(inputDataSource.columnWorldCompressionMode, x, z);
this.columnWorldCompressionMode[recipientIndex] = worldCompressionMode;
// data points //
LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z);
@@ -465,6 +483,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
*/
private static byte determineMinWorldGenStepForTwoByTwoColumn(byte[] columnGenerationSteps, int relX, int relZ)
{
// TODO merge similar logic with determineHighestWorldCompressionForTwoByTwoColumn
byte minWorldGenStepValue = Byte.MAX_VALUE;
for (int x = 0; x < 2; x++)
{
@@ -477,6 +496,25 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
}
return minWorldGenStepValue;
}
/**
* The minimum value is used because we don't want to accidentally record that
* something was generated when it wasn't.
*/
private static byte determineHighestWorldCompressionForTwoByTwoColumn(byte[] columnCompressionMode, int relX, int relZ)
{
// TODO merge similar logic with determineMinWorldGenStepForTwoByTwoColumn
byte minWorldGenStepValue = Byte.MIN_VALUE;
for (int x = 0; x < 2; x++)
{
for (int z = 0; z < 2; z++)
{
int index = relativePosToIndex(x + relX, z + relZ);
byte worldGenStepValue = columnCompressionMode[index];
minWorldGenStepValue = (byte) Math.max(minWorldGenStepValue, worldGenStepValue);
}
}
return minWorldGenStepValue;
}
private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z)
{
LongArrayList newColumnList = new LongArrayList();
@@ -817,11 +855,12 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]);
}
public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep)
public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep, EDhApiWorldCompressionMode worldCompressionMode)
{
int index = relativePosToIndex(relX, relZ);
this.dataPoints[index] = longArray;
this.columnGenerationSteps[index] = worldGenStep.value;
this.columnWorldCompressionMode[index] = worldCompressionMode.value;
if (RUN_UPDATE_DEV_VALIDATION)
@@ -863,6 +902,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
int result = this.pos.hashCode();
result = 31 * result + Arrays.deepHashCode(this.dataPoints);
result = 17 * result + Arrays.hashCode(this.columnGenerationSteps);
result = 43 * result + Arrays.hashCode(this.columnWorldCompressionMode);
this.cachedHashCode = result;
}
@@ -938,6 +978,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
}
Arrays.fill(dataSource.columnGenerationSteps, (byte) 0);
Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0);
}
return dataSource;
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+33 -3
View File
@@ -20,6 +20,7 @@
package com.seibel.distanthorizons.core.sql.dto;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
@@ -50,6 +51,8 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
/** @see EDhApiWorldGenerationStep */
public byte[] compressedColumnGenStepByteArray;
/** @see EDhApiWorldCompressionMode */
public byte[] compressedWorldCompressionModeByteArray;
public byte[] compressedMappingByteArray;
@@ -71,11 +74,12 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
{
CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum);
byte[] compressedWorldGenStepByteArray = writeGenerationStepsToBlob(dataSource.columnGenerationSteps, compressionModeEnum);
byte[] compressedWorldCompressionModeByteArray = writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, compressionModeEnum);
byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum);
return new FullDataSourceV2DTO(
dataSource.getSectionPos(),
checkedDataPointArray.checksum, compressedWorldGenStepByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray,
checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray,
dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime,
mappingByteArray, dataSource.applyToParent,
dataSource.levelMinY
@@ -84,7 +88,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
public FullDataSourceV2DTO(
DhSectionPos pos,
int dataChecksum, byte[] compressedColumnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray,
int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray,
long lastModifiedUnixDateTime, long createdUnixDateTime,
byte[] compressedMappingByteArray, boolean applyToParent,
int levelMinY)
@@ -92,6 +96,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
this.pos = pos;
this.dataChecksum = dataChecksum;
this.compressedColumnGenStepByteArray = compressedColumnGenStepByteArray;
this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray;
this.dataFormatVersion = dataFormatVersion;
this.compressionModeEnum = compressionModeEnum;
@@ -133,10 +138,11 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
{
if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion)
{
throw new IllegalStateException("There should only be one data format right now anyway.");
throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"].");
}
dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum);
dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, this.compressionModeEnum);
dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum);
dataSource.mapping.clear(dataSource.getSectionPos());
@@ -260,6 +266,30 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
}
private static byte[] writeWorldCompressionModeToBlob(byte[] worldCompressionModeByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum);
compressedOut.write(worldCompressionModeByteArray);
compressedOut.flush();
byteArrayOutputStream.close();
return byteArrayOutputStream.toByteArray();
}
private static byte[] readBlobToWorldCompressionMode(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray);
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
byte[] worldCompressionModeByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
compressedIn.readFully(worldCompressionModeByteArray);
return worldCompressionModeByteArray;
}
private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
+7 -3
View File
@@ -86,6 +86,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
byte[] dataByteArray = (byte[]) objectMap.get("Data");
byte[] columnGenStepByteArray = (byte[]) objectMap.get("ColumnGenerationStep");
byte[] columnWorldCompressionByteArray = (byte[]) objectMap.get("ColumnWorldCompressionMode");
byte[] mappingByteArray = (byte[]) objectMap.get("Mapping");
@@ -100,7 +101,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
FullDataSourceV2DTO dto = new FullDataSourceV2DTO(
pos,
dataChecksum, columnGenStepByteArray, dataFormatVersion, compressionModeEnum, dataByteArray,
dataChecksum, columnGenStepByteArray, columnWorldCompressionByteArray, dataFormatVersion, compressionModeEnum, dataByteArray,
lastModifiedUnixDateTime, createdUnixDateTime,
mappingByteArray, applyToParent,
minY);
@@ -114,13 +115,13 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
"INSERT INTO "+this.getTableName() + " (\n" +
" DetailLevel, PosX, PosZ, \n" +
" MinY, DataChecksum, \n" +
" Data, ColumnGenerationStep, Mapping, \n" +
" Data, ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" +
" DataFormatVersion, CompressionMode, ApplyToParent, \n" +
" LastModifiedUnixDateTime, CreatedUnixDateTime) \n" +
"VALUES( \n" +
" ?, ?, ?, \n" +
" ?, ?, \n" +
" ?, ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ?, ?, \n" +
" ?, ? \n" +
");";
@@ -136,6 +137,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
statement.setObject(i++, dto.compressedDataByteArray);
statement.setObject(i++, dto.compressedColumnGenStepByteArray);
statement.setObject(i++, dto.compressedWorldCompressionModeByteArray);
statement.setObject(i++, dto.compressedMappingByteArray);
statement.setObject(i++, dto.dataFormatVersion);
@@ -159,6 +161,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
" ,Data = ? \n" +
" ,ColumnGenerationStep = ? \n" +
" ,ColumnWorldCompressionMode = ? \n" +
" ,Mapping = ? \n" +
" ,DataFormatVersion = ? \n" +
@@ -177,6 +180,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
statement.setObject(i++, dto.compressedDataByteArray);
statement.setObject(i++, dto.compressedColumnGenStepByteArray);
statement.setObject(i++, dto.compressedWorldCompressionModeByteArray);
statement.setObject(i++, dto.compressedMappingByteArray);
statement.setObject(i++, dto.dataFormatVersion);
core/src/main/resources/assets/distanthorizons/lang/en_us.json
+7 -7
View File
@@ -770,18 +770,18 @@
"Full",
"distanthorizons.config.enum.EDhApiDataCompressionMode.UNCOMPRESSED":
"Uncompressed",
"1. Uncompressed",
"distanthorizons.config.enum.EDhApiDataCompressionMode.LZ4":
"LZ4",
"2. LZ4",
"distanthorizons.config.enum.EDhApiDataCompressionMode.Z_STD":
"Zstd",
"3. Zstd",
"distanthorizons.config.enum.EDhApiDataCompressionMode.LZMA2":
"LZMA2",
"4. LZMA2",
"distanthorizons.config.enum.EDhApiWorldCompressionMode.UNCOMPRESSED":
"Uncompressed",
"distanthorizons.config.enum.EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS":
"1. Merge Same Blocks",
"distanthorizons.config.enum.EDhApiWorldCompressionMode.VISUALLY_EQUAL":
"Visually Equal",
"2. Visually Equal",
"distanthorizons.config.enum.ELightGenerationMode.DISTANT_HORIZONS":
"Distant Horizons",
core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql
+1
View File
@@ -24,6 +24,7 @@ CREATE TABLE FullData (
,Data BLOB NULL
,ColumnGenerationStep BLOB NULL
,ColumnWorldCompressionMode BLOB NULL
,Mapping BLOB NULL
,DataFormatVersion TINYINT NULL