From c8f11548310017e852b9fd1bef3319b7d8d05abd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 30 Apr 2024 21:17:26 -0500 Subject: [PATCH] Remove ZStd compression option Any ZStd data will be automatically deleted and re-generated --- .../config/EDhApiDataCompressionMode.java | 8 ++-- .../distanthorizons/core/Initializer.java | 8 ++-- .../distanthorizons/core/config/Config.java | 6 --- .../core/file/AbstractDataSourceHandler.java | 13 +++++-- .../core/sql/dto/FullDataSourceV2DTO.java | 37 +++++++++++++++---- .../core/sql/repo/FullDataSourceV2Repo.java | 10 ++--- .../dataStreams/DhDataInputStream.java | 5 +-- .../dataStreams/DhDataOutputStream.java | 6 --- .../assets/distanthorizons/lang/en_us.json | 8 ++-- core/src/test/java/tests/CompressionTest.java | 12 +++--- 10 files changed, 64 insertions(+), 49 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java index ce277088c..4e093a234 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java @@ -56,14 +56,15 @@ public enum EDhApiDataCompressionMode */ LZ4(1), - /** + /* * Decent speed and good compression.

* * Read Speed: 11.78 MS / DTO
* Write Speed: 16.76 MS / DTO
* Compression ratio: 0.2199
*/ - Z_STD(2), + //@Deprecated + //Z_STD(2), /** * Extremely slow, but very good compression.

@@ -82,7 +83,8 @@ public enum EDhApiDataCompressionMode EDhApiDataCompressionMode(int value) { this.value = (byte) value; } - public static EDhApiDataCompressionMode getFromValue(byte value) + /** @throws IllegalArgumentException if the value doesn't map to a value */ + public static EDhApiDataCompressionMode getFromValue(byte value) throws IllegalArgumentException { EDhApiDataCompressionMode[] enumList = EDhApiDataCompressionMode.values(); for (int i = 0; i < enumList.length; i++) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index ede8a114b..76c2f0515 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -28,10 +28,10 @@ import com.seibel.distanthorizons.core.api.external.methods.config.DhApiConfig; import com.seibel.distanthorizons.core.api.external.methods.data.DhApiTerrainDataRepo; import com.seibel.distanthorizons.api.DhApi; import com.seibel.distanthorizons.core.render.DhApiRenderProxy; -//import io.netty.buffer.ByteBuf; import net.jpountz.lz4.LZ4FrameOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.tukaani.xz.XZOutputStream; import java.awt.*; @@ -47,15 +47,17 @@ public class Initializer { // if any library isn't present in the jar its class // will throw an error (not an exception) - Class compressor = LZ4FrameOutputStream.class; + Class fastCompressor = LZ4FrameOutputStream.class; + Class smallCompressor = XZOutputStream.class; //Class networking = ByteBuf.class; - Class toml = com.electronwill.nightconfig.core.Config.class; + Class config = com.electronwill.nightconfig.core.Config.class; Class oldFastUtil = it.unimi.dsi.fastutil.longs.LongArrayList.class; // available in 8.2.1 //Class newFastUtil = it.unimi.dsi.fastutil.ints.IntUnaryOperator.class; // available in 8.5.13 } catch (Throwable e) { LOGGER.fatal("Critical programmer error: One or more libraries aren't present. Error: [" + e.getMessage() + "]."); + // throwing here should crash the game, notifying the developer that something is wrong throw e; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index ac2f0bb33..0aaa4b94a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -771,12 +771,6 @@ public class Config + "Estimated average DTO read speed: 1.85 ms\n" + "Estimated average DTO write speed: 9.46 ms\n" + "\n" - + EDhApiDataCompressionMode.Z_STD + " \n" - + "A good middle ground between speed and compression.\n" - + "Expected Compression Ratio: 0.21\n" - + "Estimated average DTO read speed: 11.78 ms\n" - + "Estimated average DTO write speed: 16.77 ms\n" - + "\n" + EDhApiDataCompressionMode.LZMA2 + " \n" + "Slow but very good compression.\n" + "Expected Compression Ratio: 0.14\n" diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index c89168b75..207068e2b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -33,6 +33,8 @@ public abstract class AbstractDataSourceHandler implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final Set CORRUPT_DATA_ERRORS_LOGGED = Collections.newSetFromMap(new ConcurrentHashMap<>()); + /** * The highest numerical detail level possible. @@ -154,9 +156,14 @@ public abstract class AbstractDataSourceHandler } catch (DataCorruptedException e) { - // stack trace not included since a lot of corrupt data would cause the log to get quite messy, - // and it should be fairly easy to see what the problem was from the message - LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage()); + // Only log each message type once. + // This is done to prevent logging "No compression mode with the value [2]" 10,000 times + // if the user is migrating from a nightly build and used ZStd. + if (CORRUPT_DATA_ERRORS_LOGGED.add(e.getMessage())) + { + LOGGER.warn("Corrupted data found at pos " + pos + ". Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: " + e.getMessage(), e); + } + this.repo.deleteWithKey(pos); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index bdab7dd0d..728a6d350 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -60,7 +60,7 @@ public class FullDataSourceV2DTO implements IBaseDTO public byte[] compressedMappingByteArray; public byte dataFormatVersion; - public EDhApiDataCompressionMode compressionModeEnum; + public byte compressionModeValue; public boolean applyToParent; @@ -82,7 +82,7 @@ public class FullDataSourceV2DTO implements IBaseDTO return new FullDataSourceV2DTO( dataSource.getPos(), - checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, + checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum.value, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, dataSource.levelMinY @@ -91,7 +91,7 @@ public class FullDataSourceV2DTO implements IBaseDTO public FullDataSourceV2DTO( DhSectionPos pos, - int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray, + int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, byte compressionModeValue, byte[] compressedDataByteArray, long lastModifiedUnixDateTime, long createdUnixDateTime, byte[] compressedMappingByteArray, boolean applyToParent, int levelMinY) @@ -102,7 +102,7 @@ public class FullDataSourceV2DTO implements IBaseDTO this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray; this.dataFormatVersion = dataFormatVersion; - this.compressionModeEnum = compressionModeEnum; + this.compressionModeValue = compressionModeValue; this.compressedDataByteArray = compressedDataByteArray; this.compressedMappingByteArray = compressedMappingByteArray; @@ -144,9 +144,23 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"]."); } - dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); - dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, this.compressionModeEnum); - dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); + + EDhApiDataCompressionMode compressionModeEnum; + try + { + compressionModeEnum = this.getCompressionMode(); + } + catch (IllegalArgumentException e) + { + // may happen if ZStd was used (which was added and removed during the nightly builds) + // or if the compressor value is changed to an invalid option + throw new DataCorruptedException(e); + } + + + dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, compressionModeEnum); + dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, compressionModeEnum); + dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, compressionModeEnum); dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test @@ -157,7 +171,7 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests."); } - dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, this.compressionModeEnum)); + dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, compressionModeEnum)); } dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; @@ -341,6 +355,13 @@ public class FullDataSourceV2DTO implements IBaseDTO + //================// + // helper methods // + //================// + + public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } + + //================// // helper classes // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 619581a14..7f3df8519 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -24,6 +24,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; @@ -91,8 +92,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo