add (native) ZStd compression as default compressor
This commit is contained in:
James Seibel
+8
-9
@@ -55,15 +55,14 @@ public enum EDhApiDataCompressionMode
|
||||
*/
|
||||
LZ4(1),
|
||||
|
||||
///**
|
||||
// * Decent speed and good compression. <br><br>
|
||||
// *
|
||||
// * Read Speed: 9.31 MS / DTO <br>
|
||||
// * Write Speed: 15.13 MS / DTO <br>
|
||||
// * Compression ratio: 0.2606 <br>
|
||||
// */
|
||||
////@DisallowSelectingViaConfigGui
|
||||
//Z_STD(2),
|
||||
/**
|
||||
* Decent speed and good compression. <br><br>
|
||||
*
|
||||
* Read Speed: 9.31 MS / DTO <br>
|
||||
* Write Speed: 15.13 MS / DTO <br>
|
||||
* Compression ratio: 0.2606 <br>
|
||||
*/
|
||||
Z_STD(2),
|
||||
|
||||
|
||||
/**
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
package com.seibel.distanthorizons.core;
|
||||
|
||||
import com.github.luben.zstd.ZstdOutputStream;
|
||||
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
|
||||
import com.seibel.distanthorizons.core.render.renderer.generic.GenericRenderObjectFactory;
|
||||
import com.seibel.distanthorizons.core.sql.DatabaseUpdater;
|
||||
@@ -54,8 +55,9 @@ public class Initializer
|
||||
{
|
||||
// if any library isn't present in the jar its class
|
||||
// will throw an error (not an exception)
|
||||
Class<?> fastCompressor = LZ4FrameOutputStream.class;
|
||||
Class<?> smallCompressor = XZOutputStream.class;
|
||||
Class<?> lz4Compressor = LZ4FrameOutputStream.class;
|
||||
Class<?> zstdCompressor = ZstdOutputStream.class;
|
||||
Class<?> lzmaCompressor = XZOutputStream.class;
|
||||
//Class<?> networking = ByteBuf.class;
|
||||
Class<?> config = com.electronwill.nightconfig.core.Config.class;
|
||||
Class<?> oldFastUtil = it.unimi.dsi.fastutil.longs.LongArrayList.class; // available in 8.2.1
|
||||
@@ -63,30 +65,6 @@ public class Initializer
|
||||
Class<?> sqliteJava = org.sqlite.SQLiteConnection.class;
|
||||
Class<?> sqliteNative = org.sqlite.core.NativeDB.class;
|
||||
|
||||
//// maybe these lines are needed to shade SQLite, James isn't sure.
|
||||
//// Although they never seemed to fail, which is a bit odd.
|
||||
//try
|
||||
//{
|
||||
// // needed by Forge to load the Java database connection
|
||||
// Class.forName("org.sqlite.JDBC");
|
||||
// LOGGER.info("loaded normal SQLITE");
|
||||
//}
|
||||
//catch (ClassNotFoundException e)
|
||||
//{
|
||||
// LOGGER.warn("normal: " + e.getMessage(), e);
|
||||
//}
|
||||
//
|
||||
//try
|
||||
//{
|
||||
// // needed by Forge to load the Java database connection
|
||||
// Class.forName("DistantHorizons.libraries.sqlite.JDBC");
|
||||
// LOGGER.info("loaded shaded SQLITE");
|
||||
//}
|
||||
//catch (ClassNotFoundException e)
|
||||
//{
|
||||
// LOGGER.warn("shaded: " + e.getMessage(), e);
|
||||
//}
|
||||
|
||||
boolean sqliteLoaded = SQLiteJDBCLoader.initialize();
|
||||
if (!sqliteLoaded)
|
||||
{
|
||||
|
||||
@@ -1332,7 +1332,7 @@ public class Config
|
||||
.build();
|
||||
|
||||
public static ConfigEntry<EDhApiDataCompressionMode> dataCompression = new ConfigEntry.Builder<EDhApiDataCompressionMode>()
|
||||
.set(EDhApiDataCompressionMode.LZMA2)
|
||||
.set(EDhApiDataCompressionMode.Z_STD)
|
||||
.comment(""
|
||||
+ "What algorithm should be used to compress new LOD data? \n"
|
||||
+ "This setting will only affect new or updated LOD data, \n"
|
||||
@@ -1342,14 +1342,20 @@ public class Config
|
||||
+ EDhApiDataCompressionMode.UNCOMPRESSED + " \n"
|
||||
+ "Should only be used for testing, is worse in every way vs ["+EDhApiDataCompressionMode.LZ4+"].\n"
|
||||
+ "Expected Compression Ratio: 1.0\n"
|
||||
+ "Estimated average DTO read speed: 3.25 milliseconds\n"
|
||||
+ "Estimated average DTO write speed: 5.99 milliseconds\n"
|
||||
+ "Estimated average DTO read speed: 6.09 milliseconds\n"
|
||||
+ "Estimated average DTO write speed: 6.01 milliseconds\n"
|
||||
+ "\n"
|
||||
+ EDhApiDataCompressionMode.LZ4 + " \n"
|
||||
+ "A good option if you're CPU limited and have plenty of hard drive space.\n"
|
||||
+ "Expected Compression Ratio: 0.26\n"
|
||||
+ "Estimated average DTO read speed: 1.85 ms\n"
|
||||
+ "Estimated average DTO write speed: 9.46 ms\n"
|
||||
+ "Expected Compression Ratio: 0.4513\n"
|
||||
+ "Estimated average DTO read speed: 3.25 ms\n"
|
||||
+ "Estimated average DTO write speed: 5.99 ms\n"
|
||||
+ "\n"
|
||||
+ EDhApiDataCompressionMode.Z_STD + " \n"
|
||||
+ "A good option if you're CPU limited and have plenty of hard drive space.\n"
|
||||
+ "Expected Compression Ratio: 0.2606\n"
|
||||
+ "Estimated average DTO read speed: 9.31 ms\n"
|
||||
+ "Estimated average DTO write speed: 15.13 ms\n"
|
||||
+ "\n"
|
||||
+ EDhApiDataCompressionMode.LZMA2 + " \n"
|
||||
+ "Slow but very good compression.\n"
|
||||
|
||||
+4
-4
@@ -19,9 +19,10 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.util.objects.dataStreams;
|
||||
|
||||
import com.github.luben.zstd.RecyclingBufferPool;
|
||||
import com.github.luben.zstd.ZstdInputStream;
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import net.jpountz.lz4.LZ4FrameInputStream;
|
||||
//import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.tukaani.xz.ResettableArrayCache;
|
||||
@@ -42,7 +43,6 @@ import java.io.*;
|
||||
public class DhDataInputStream extends DataInputStream
|
||||
{
|
||||
private static final ThreadLocal<ResettableArrayCache> LZMA_RESETTABLE_ARRAY_CACHE_GETTER = ThreadLocal.withInitial(() -> new ResettableArrayCache(new LzmaArrayCache()));
|
||||
//private static final ThreadLocal<ZstdArrayCache> ZSTD_RESETTABLE_ARRAY_CACHE_GETTER = ThreadLocal.withInitial(() -> new ZstdArrayCache());
|
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger();
|
||||
|
||||
@@ -61,8 +61,8 @@ public class DhDataInputStream extends DataInputStream
|
||||
return stream;
|
||||
case LZ4:
|
||||
return new LZ4FrameInputStream(stream);
|
||||
//case Z_STD:
|
||||
// return new ZstdCompressorInputStream(stream, ZSTD_RESETTABLE_ARRAY_CACHE_GETTER.get());
|
||||
case Z_STD:
|
||||
return new ZstdInputStream(stream, RecyclingBufferPool.INSTANCE);
|
||||
case LZMA2:
|
||||
// using an array cache significantly reduces GC pressure
|
||||
ResettableArrayCache arrayCache = LZMA_RESETTABLE_ARRAY_CACHE_GETTER.get();
|
||||
|
||||
+3
-3
@@ -19,11 +19,11 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.util.objects.dataStreams;
|
||||
|
||||
import com.github.luben.zstd.ZstdOutputStream;
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import net.jpountz.lz4.LZ4Factory;
|
||||
import net.jpountz.lz4.LZ4FrameOutputStream;
|
||||
import net.jpountz.xxhash.XXHashFactory;
|
||||
//import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.tukaani.xz.*;
|
||||
@@ -55,8 +55,8 @@ public class DhDataOutputStream extends DataOutputStream
|
||||
case UNCOMPRESSED:
|
||||
return stream;
|
||||
|
||||
//case Z_STD:
|
||||
// return new ZstdCompressorOutputStream(stream, 3, true, true);
|
||||
case Z_STD:
|
||||
return new ZstdOutputStream(stream, 3, true, true);
|
||||
case LZ4:
|
||||
return new LZ4FrameOutputStream(stream,
|
||||
LZ4FrameOutputStream.BLOCKSIZE.SIZE_64KB, -1L,
|
||||
|
||||
Reference in New Issue
Block a user