wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove ZStd compression option

Browse Source 
Any ZStd data will be automatically deleted and re-generated
This commit is contained in:
James Seibel
2024-04-30 21:17:26 -05:00
parent 9196480e50
commit c8f1154831
10 changed files with 64 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java
+5 -3
View File
@@ -56,14 +56,15 @@ public enum EDhApiDataCompressionMode
*/
LZ4(1),
/**
/*
* Decent speed and good compression. <br><br>
*
* Read Speed: 11.78 MS / DTO <br>
* Write Speed: 16.76 MS / DTO <br>
* Compression ratio: 0.2199 <br>
*/
Z_STD(2),
//@Deprecated
//Z_STD(2),
/**
* Extremely slow, but very good compression. <br><br>
@@ -82,7 +83,8 @@ public enum EDhApiDataCompressionMode
EDhApiDataCompressionMode(int value) { this.value = (byte) value; }
public static EDhApiDataCompressionMode getFromValue(byte value)
/** @throws IllegalArgumentException if the value doesn't map to a value */
public static EDhApiDataCompressionMode getFromValue(byte value) throws IllegalArgumentException
{
EDhApiDataCompressionMode[] enumList = EDhApiDataCompressionMode.values();
for (int i = 0; i < enumList.length; i++)
core/src/main/java/com/seibel/distanthorizons/core/Initializer.java
+5 -3
View File
@@ -28,10 +28,10 @@ import com.seibel.distanthorizons.core.api.external.methods.config.DhApiConfig;
import com.seibel.distanthorizons.core.api.external.methods.data.DhApiTerrainDataRepo;
import com.seibel.distanthorizons.api.DhApi;
import com.seibel.distanthorizons.core.render.DhApiRenderProxy;
//import io.netty.buffer.ByteBuf;
import net.jpountz.lz4.LZ4FrameOutputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.tukaani.xz.XZOutputStream;
import java.awt.*;
@@ -47,15 +47,17 @@ public class Initializer
{
// if any library isn't present in the jar its class
// will throw an error (not an exception)
Class<?> compressor = LZ4FrameOutputStream.class;
Class<?> fastCompressor = LZ4FrameOutputStream.class;
Class<?> smallCompressor = XZOutputStream.class;
//Class<?> networking = ByteBuf.class;
Class<?> toml = com.electronwill.nightconfig.core.Config.class;
Class<?> config = com.electronwill.nightconfig.core.Config.class;
Class<?> oldFastUtil = it.unimi.dsi.fastutil.longs.LongArrayList.class; // available in 8.2.1
//Class<?> newFastUtil = it.unimi.dsi.fastutil.ints.IntUnaryOperator.class; // available in 8.5.13
}
catch (Throwable e)
{
LOGGER.fatal("Critical programmer error: One or more libraries aren't present. Error: [" + e.getMessage() + "].");
// throwing here should crash the game, notifying the developer that something is wrong
throw e;
}
core/src/main/java/com/seibel/distanthorizons/core/config/Config.java
-6
View File
@@ -771,12 +771,6 @@ public class Config
+ "Estimated average DTO read speed: 1.85 ms\n"
+ "Estimated average DTO write speed: 9.46 ms\n"
+ "\n"
+ EDhApiDataCompressionMode.Z_STD + " \n"
+ "A good middle ground between speed and compression.\n"
+ "Expected Compression Ratio: 0.21\n"
+ "Estimated average DTO read speed: 11.78 ms\n"
+ "Estimated average DTO write speed: 16.77 ms\n"
+ "\n"
+ EDhApiDataCompressionMode.LZMA2 + " \n"
+ "Slow but very good compression.\n"
+ "Expected Compression Ratio: 0.14\n"
core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java
+10 -3
View File
@@ -33,6 +33,8 @@ public abstract class AbstractDataSourceHandler
implements AutoCloseable
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final Set<String> CORRUPT_DATA_ERRORS_LOGGED = Collections.newSetFromMap(new ConcurrentHashMap<>());
/**
* The highest numerical detail level possible.
@@ -154,9 +156,14 @@ public abstract class AbstractDataSourceHandler
}
catch (DataCorruptedException e)
{
// stack trace not included since a lot of corrupt data would cause the log to get quite messy,
// and it should be fairly easy to see what the problem was from the message
LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage());
// Only log each message type once.
// This is done to prevent logging "No compression mode with the value [2]" 10,000 times
// if the user is migrating from a nightly build and used ZStd.
if (CORRUPT_DATA_ERRORS_LOGGED.add(e.getMessage()))
{
LOGGER.warn("Corrupted data found at pos " + pos + ". Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: " + e.getMessage(), e);
}
this.repo.deleteWithKey(pos);
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+29 -8
View File
@@ -60,7 +60,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
public byte[] compressedMappingByteArray;
public byte dataFormatVersion;
public EDhApiDataCompressionMode compressionModeEnum;
public byte compressionModeValue;
public boolean applyToParent;
@@ -82,7 +82,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
return new FullDataSourceV2DTO(
dataSource.getPos(),
checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray,
checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum.value, checkedDataPointArray.byteArray,
dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime,
mappingByteArray, dataSource.applyToParent,
dataSource.levelMinY
@@ -91,7 +91,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
public FullDataSourceV2DTO(
DhSectionPos pos,
int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray,
int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, byte compressionModeValue, byte[] compressedDataByteArray,
long lastModifiedUnixDateTime, long createdUnixDateTime,
byte[] compressedMappingByteArray, boolean applyToParent,
int levelMinY)
@@ -102,7 +102,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray;
this.dataFormatVersion = dataFormatVersion;
this.compressionModeEnum = compressionModeEnum;
this.compressionModeValue = compressionModeValue;
this.compressedDataByteArray = compressedDataByteArray;
this.compressedMappingByteArray = compressedMappingByteArray;
@@ -144,9 +144,23 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"].");
}
dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum);
dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, this.compressionModeEnum);
dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum);
EDhApiDataCompressionMode compressionModeEnum;
try
{
compressionModeEnum = this.getCompressionMode();
}
catch (IllegalArgumentException e)
{
// may happen if ZStd was used (which was added and removed during the nightly builds)
// or if the compressor value is changed to an invalid option
throw new DataCorruptedException(e);
}
dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, compressionModeEnum);
dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, compressionModeEnum);
dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, compressionModeEnum);
dataSource.mapping.clear(dataSource.getPos());
// should only be null when used in a unit test
@@ -157,7 +171,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests.");
}
dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, this.compressionModeEnum));
dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, compressionModeEnum));
}
dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime;
@@ -341,6 +355,13 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
//================//
// helper methods //
//================//
public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); }
//================//
// helper classes //
//================//
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
+5 -5
View File
@@ -24,6 +24,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
@@ -91,8 +92,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
byte dataFormatVersion = (Byte) objectMap.get("DataFormatVersion");
byte compressionMode = (Byte) objectMap.get("CompressionMode");
EDhApiDataCompressionMode compressionModeEnum = EDhApiDataCompressionMode.getFromValue(compressionMode);
byte compressionModeValue = (Byte) objectMap.get("CompressionMode");
boolean applyToParent = ((int) objectMap.get("ApplyToParent")) == 1;
@@ -101,7 +101,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
FullDataSourceV2DTO dto = new FullDataSourceV2DTO(
pos,
dataChecksum, columnGenStepByteArray, columnWorldCompressionByteArray, dataFormatVersion, compressionModeEnum, dataByteArray,
dataChecksum, columnGenStepByteArray, columnWorldCompressionByteArray, dataFormatVersion, compressionModeValue, dataByteArray,
lastModifiedUnixDateTime, createdUnixDateTime,
mappingByteArray, applyToParent,
minY);
@@ -141,7 +141,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
statement.setObject(i++, dto.compressedMappingByteArray);
statement.setObject(i++, dto.dataFormatVersion);
statement.setObject(i++, dto.compressionModeEnum.value);
statement.setObject(i++, dto.compressionModeValue);
statement.setObject(i++, dto.applyToParent);
statement.setObject(i++, System.currentTimeMillis()); // last modified unix time
@@ -184,7 +184,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<DhSectionPos, FullDataS
statement.setObject(i++, dto.compressedMappingByteArray);
statement.setObject(i++, dto.dataFormatVersion);
statement.setObject(i++, dto.compressionModeEnum.value);
statement.setObject(i++, dto.compressionModeValue);
statement.setObject(i++, dto.applyToParent);
statement.setObject(i++, System.currentTimeMillis()); // last modified unix time
core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java
+1 -4
View File
@@ -19,9 +19,8 @@
package com.seibel.distanthorizons.core.util.objects.dataStreams;
import com.github.luben.zstd.RecyclingBufferPool;
import com.github.luben.zstd.ZstdInputStream;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import net.jpountz.lz4.LZ4FrameInputStream;
import org.tukaani.xz.XZInputStream;
@@ -51,8 +50,6 @@ public class DhDataInputStream extends DataInputStream
return stream;
case LZ4:
return new LZ4FrameInputStream(stream);
case Z_STD:
return new ZstdInputStream(stream, RecyclingBufferPool.INSTANCE);
case LZMA2:
// Note: all LZMA/XZ compressors can be decompressed using this same InputStream
return new XZInputStream(stream);
core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java
-6
View File
@@ -19,14 +19,10 @@
package com.seibel.distanthorizons.core.util.objects.dataStreams;
import com.github.luben.zstd.RecyclingBufferPool;
import com.github.luben.zstd.ZstdOutputStream;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.lz4.LZ4FrameOutputStream;
import net.jpountz.xxhash.XXHashFactory;
import org.apache.logging.log4j.Logger;
import org.tukaani.xz.*;
import java.io.*;
@@ -59,8 +55,6 @@ public class DhDataOutputStream extends DataOutputStream
LZ4Factory.nativeInstance().fastCompressor(),
XXHashFactory.nativeInstance().hash32(),
LZ4FrameOutputStream.FLG.Bits.BLOCK_INDEPENDENCE);
case Z_STD:
return new ZstdOutputStream(stream, RecyclingBufferPool.INSTANCE);
case LZMA2:
// using an array cache significantly reduces GC pressure
ResettableArrayCache arrayCache = LZMA_RESETTABLE_ARRAY_CACHE_GETTER.get();
core/src/main/resources/assets/distanthorizons/lang/en_us.json
+3 -5
View File
@@ -776,13 +776,11 @@
"Full",
"distanthorizons.config.enum.EDhApiDataCompressionMode.UNCOMPRESSED":
"1. Uncompressed",
"Uncompressed",
"distanthorizons.config.enum.EDhApiDataCompressionMode.LZ4":
"2. LZ4",
"distanthorizons.config.enum.EDhApiDataCompressionMode.Z_STD":
"3. Zstd",
"Fast/Big - LZ4",
"distanthorizons.config.enum.EDhApiDataCompressionMode.LZMA2":
"4. LZMA2",
"Slow/Small - LZMA2",
"distanthorizons.config.enum.EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS":
"1. Merge Same Blocks",
core/src/test/java/tests/CompressionTest.java
+6 -6
View File
@@ -213,11 +213,11 @@ public class CompressionTest
}
//@Test
public void Zstd() // middle of the road
{
String compressorName = "Zstd";
this.testCompressor(compressorName, EDhApiDataCompressionMode.Z_STD);
}
//public void Zstd() // middle of the road
//{
// String compressorName = "Zstd";
// this.testCompressor(compressorName, EDhApiDataCompressionMode.Z_STD);
//}
//@Test
public void LZMA2() // very slow, very good compression though
@@ -293,7 +293,7 @@ public class CompressionTest
// uncompressed input //
FullDataSourceV2DTO uncompressedDto = uncompressedRepo.getByKey(pos);
Assert.assertEquals(uncompressedDto.compressionModeEnum, EDhApiDataCompressionMode.UNCOMPRESSED);
Assert.assertEquals(uncompressedDto.compressionModeValue, EDhApiDataCompressionMode.UNCOMPRESSED.value);
FullDataSourceV2 uncompressedDataSource = uncompressedDto.createUnitTestDataSource();
long uncompressedDtoSize = uncompressedRepo.getDataSizeInBytes(pos);