wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add (native) ZStd compression as default compressor

Browse Source
This commit is contained in:
James Seibel
2025-09-03 07:39:58 -05:00
parent 674fc30e77
commit 5b41c7d48a
5 changed files with 31 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/Initializer.java
+4 -26
View File
@@ -19,6 +19,7 @@
package com.seibel.distanthorizons.core;
import com.github.luben.zstd.ZstdOutputStream;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.render.renderer.generic.GenericRenderObjectFactory;
import com.seibel.distanthorizons.core.sql.DatabaseUpdater;
@@ -54,8 +55,9 @@ public class Initializer
{
// if any library isn't present in the jar its class
// will throw an error (not an exception)
Class<?> fastCompressor = LZ4FrameOutputStream.class;
Class<?> smallCompressor = XZOutputStream.class;
Class<?> lz4Compressor = LZ4FrameOutputStream.class;
Class<?> zstdCompressor = ZstdOutputStream.class;
Class<?> lzmaCompressor = XZOutputStream.class;
//Class<?> networking = ByteBuf.class;
Class<?> config = com.electronwill.nightconfig.core.Config.class;
Class<?> oldFastUtil = it.unimi.dsi.fastutil.longs.LongArrayList.class; // available in 8.2.1
@@ -63,30 +65,6 @@ public class Initializer
Class<?> sqliteJava = org.sqlite.SQLiteConnection.class;
Class<?> sqliteNative = org.sqlite.core.NativeDB.class;
//// maybe these lines are needed to shade SQLite, James isn't sure.
//// Although they never seemed to fail, which is a bit odd.
//try
//{
// // needed by Forge to load the Java database connection
// Class.forName("org.sqlite.JDBC");
// LOGGER.info("loaded normal SQLITE");
//}
//catch (ClassNotFoundException e)
//{
// LOGGER.warn("normal: " + e.getMessage(), e);
//}
//
//try
//{
// // needed by Forge to load the Java database connection
// Class.forName("DistantHorizons.libraries.sqlite.JDBC");
// LOGGER.info("loaded shaded SQLITE");
//}
//catch (ClassNotFoundException e)
//{
// LOGGER.warn("shaded: " + e.getMessage(), e);
//}
boolean sqliteLoaded = SQLiteJDBCLoader.initialize();
if (!sqliteLoaded)
{
core/src/main/java/com/seibel/distanthorizons/core/config/Config.java
+12 -6
View File
@@ -1332,7 +1332,7 @@ public class Config
.build();
public static ConfigEntry<EDhApiDataCompressionMode> dataCompression = new ConfigEntry.Builder<EDhApiDataCompressionMode>()
.set(EDhApiDataCompressionMode.LZMA2)
.set(EDhApiDataCompressionMode.Z_STD)
.comment(""
+ "What algorithm should be used to compress new LOD data? \n"
+ "This setting will only affect new or updated LOD data, \n"
@@ -1342,14 +1342,20 @@ public class Config
+ EDhApiDataCompressionMode.UNCOMPRESSED + " \n"
+ "Should only be used for testing, is worse in every way vs ["+EDhApiDataCompressionMode.LZ4+"].\n"
+ "Expected Compression Ratio: 1.0\n"
+ "Estimated average DTO read speed: 3.25 milliseconds\n"
+ "Estimated average DTO write speed: 5.99 milliseconds\n"
+ "Estimated average DTO read speed: 6.09 milliseconds\n"
+ "Estimated average DTO write speed: 6.01 milliseconds\n"
+ "\n"
+ EDhApiDataCompressionMode.LZ4 + " \n"
+ "A good option if you're CPU limited and have plenty of hard drive space.\n"
+ "Expected Compression Ratio: 0.26\n"
+ "Estimated average DTO read speed: 1.85 ms\n"
+ "Estimated average DTO write speed: 9.46 ms\n"
+ "Expected Compression Ratio: 0.4513\n"
+ "Estimated average DTO read speed: 3.25 ms\n"
+ "Estimated average DTO write speed: 5.99 ms\n"
+ "\n"
+ EDhApiDataCompressionMode.Z_STD + " \n"
+ "A good option if you're CPU limited and have plenty of hard drive space.\n"
+ "Expected Compression Ratio: 0.2606\n"
+ "Estimated average DTO read speed: 9.31 ms\n"
+ "Estimated average DTO write speed: 15.13 ms\n"
+ "\n"
+ EDhApiDataCompressionMode.LZMA2 + " \n"
+ "Slow but very good compression.\n"
core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java
+4 -4
View File
@@ -19,9 +19,10 @@
package com.seibel.distanthorizons.core.util.objects.dataStreams;
import com.github.luben.zstd.RecyclingBufferPool;
import com.github.luben.zstd.ZstdInputStream;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import net.jpountz.lz4.LZ4FrameInputStream;
//import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.tukaani.xz.ResettableArrayCache;
@@ -42,7 +43,6 @@ import java.io.*;
public class DhDataInputStream extends DataInputStream
{
private static final ThreadLocal<ResettableArrayCache> LZMA_RESETTABLE_ARRAY_CACHE_GETTER = ThreadLocal.withInitial(() -> new ResettableArrayCache(new LzmaArrayCache()));
//private static final ThreadLocal<ZstdArrayCache> ZSTD_RESETTABLE_ARRAY_CACHE_GETTER = ThreadLocal.withInitial(() -> new ZstdArrayCache());
private static final Logger LOGGER = LogManager.getLogger();
@@ -61,8 +61,8 @@ public class DhDataInputStream extends DataInputStream
return stream;
case LZ4:
return new LZ4FrameInputStream(stream);
//case Z_STD:
// return new ZstdCompressorInputStream(stream, ZSTD_RESETTABLE_ARRAY_CACHE_GETTER.get());
case Z_STD:
return new ZstdInputStream(stream, RecyclingBufferPool.INSTANCE);
case LZMA2:
// using an array cache significantly reduces GC pressure
ResettableArrayCache arrayCache = LZMA_RESETTABLE_ARRAY_CACHE_GETTER.get();
core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java
+3 -3
View File
@@ -19,11 +19,11 @@
package com.seibel.distanthorizons.core.util.objects.dataStreams;
import com.github.luben.zstd.ZstdOutputStream;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.lz4.LZ4FrameOutputStream;
import net.jpountz.xxhash.XXHashFactory;
//import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.tukaani.xz.*;
@@ -55,8 +55,8 @@ public class DhDataOutputStream extends DataOutputStream
case UNCOMPRESSED:
return stream;
//case Z_STD:
// return new ZstdCompressorOutputStream(stream, 3, true, true);
case Z_STD:
return new ZstdOutputStream(stream, 3, true, true);
case LZ4:
return new LZ4FrameOutputStream(stream,
LZ4FrameOutputStream.BLOCKSIZE.SIZE_64KB, -1L,