Add multiple compression options and unit tests
This commit is contained in:
James Seibel
+100
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* This file is part of the Distant Horizons mod
|
||||
* licensed under the GNU LGPL v3 License.
|
||||
*
|
||||
* Copyright (C) 2020-2023 James Seibel
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.seibel.distanthorizons.api.enums.config;
|
||||
|
||||
/**
|
||||
* UNCOMPRESSED <br>
|
||||
* LZ4 <br>
|
||||
* ZSTD <br>
|
||||
* XZ <br><br>
|
||||
*
|
||||
* Note: speed and compression ratios are examples
|
||||
* and should only be used for estimated comparisons.
|
||||
*
|
||||
* @version 2024-3-16
|
||||
* @since API 1.1.0
|
||||
*/
|
||||
public enum EDhApiDataCompressionMode
|
||||
{
|
||||
// Reminder:
|
||||
// when adding items up the API minor version
|
||||
// when removing items up the API major version
|
||||
|
||||
/**
|
||||
* Should only be used internally and for unit testing. <br><br>
|
||||
*
|
||||
* Read Speed: 1.64 MS / DTO <br>
|
||||
* Write Speed: 12.44 MS / DTO <br>
|
||||
* Compression ratio: 1.0 <br>
|
||||
*/
|
||||
@DisallowSelectingViaConfigGui
|
||||
UNCOMPRESSED(0),
|
||||
|
||||
/**
|
||||
* Extremely fast (often faster than uncompressed), but generally poor compression. <br><br>
|
||||
*
|
||||
* Read Speed: 1.85 MS / DTO <br>
|
||||
* Write Speed: 9.46 MS / DTO <br>
|
||||
* Compression ratio: 0.3638 <br>
|
||||
*/
|
||||
LZ4(1),
|
||||
|
||||
/**
|
||||
* Decent speed and good compression. <br><br>
|
||||
*
|
||||
* Read Speed: 11.78 MS / DTO <br>
|
||||
* Write Speed: 16.76 MS / DTO <br>
|
||||
* Compression ratio: 0.2199 <br>
|
||||
*/
|
||||
Z_STD(2),
|
||||
|
||||
/**
|
||||
* Extremely slow, but very good compression. <br><br>
|
||||
*
|
||||
* Read Speed: 12.25 MS / DTO <br>
|
||||
* Write Speed: 490.07 MS / DTO <br>
|
||||
* Compression ratio: 0.1242 <br>
|
||||
*/
|
||||
LZMA2(3);
|
||||
|
||||
|
||||
|
||||
/** More stable than using the ordinal of the enum */
|
||||
public final byte value;
|
||||
|
||||
EDhApiDataCompressionMode(int value) { this.value = (byte) value; }
|
||||
|
||||
|
||||
public static EDhApiDataCompressionMode getFromValue(byte value)
|
||||
{
|
||||
EDhApiDataCompressionMode[] enumList = EDhApiDataCompressionMode.values();
|
||||
for (int i = 0; i < enumList.length; i++)
|
||||
{
|
||||
if (enumList[i].value == value)
|
||||
{
|
||||
return enumList[i];
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException("No compression mode with the value ["+value+"]");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -801,6 +801,40 @@ public class Config
|
||||
+ "")
|
||||
.build();
|
||||
|
||||
public static ConfigEntry<EDhApiDataCompressionMode> dataCompression = new ConfigEntry.Builder<EDhApiDataCompressionMode>()
|
||||
.set(EDhApiDataCompressionMode.LZMA2)
|
||||
.comment(""
|
||||
+ "What algorithm should be used to compress new LOD data? \n"
|
||||
+ "This setting will only affect new or updated LOD data, \n"
|
||||
+ "any data already generated when this setting is changed will be\n"
|
||||
+ "unaffected until it needs to be re-written to the database.\n"
|
||||
+ "\n"
|
||||
+ EDhApiDataCompressionMode.UNCOMPRESSED + " \n"
|
||||
+ "Should only be used for testing, is worse in every way vs ["+EDhApiDataCompressionMode.LZ4+"].\n"
|
||||
+ "Expected Compression Ratio: 1.0\n"
|
||||
+ "Estimated average DTO read speed: 1.64 milliseconds\n"
|
||||
+ "Estimated average DTO write speed: 12.44 milliseconds\n"
|
||||
+ "\n"
|
||||
+ EDhApiDataCompressionMode.LZ4 + " \n"
|
||||
+ "A good option if you're CPU limited and have plenty of hard drive space.\n"
|
||||
+ "Expected Compression Ratio: 0.36\n"
|
||||
+ "Estimated average DTO read speed: 1.85 ms\n"
|
||||
+ "Estimated average DTO write speed: 9.46 ms\n"
|
||||
+ "\n"
|
||||
+ EDhApiDataCompressionMode.Z_STD + " \n"
|
||||
+ "A good middle ground between speed and compression.\n"
|
||||
+ "Expected Compression Ratio: 0.21\n"
|
||||
+ "Estimated average DTO read speed: 11.78 ms\n"
|
||||
+ "Estimated average DTO write speed: 16.77 ms\n"
|
||||
+ "\n"
|
||||
+ EDhApiDataCompressionMode.LZMA2 + " \n"
|
||||
+ "Slow but very good compression.\n"
|
||||
+ "Expected Compression Ratio: 0.14\n"
|
||||
+ "Estimated average DTO read speed: 11.89 ms\n"
|
||||
+ "Estimated average DTO write speed: 192.01 ms\n"
|
||||
+ "")
|
||||
.build();
|
||||
|
||||
}
|
||||
|
||||
public static class Multiplayer
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package com.seibel.distanthorizons.core.file;
|
||||
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource;
|
||||
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
|
||||
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
|
||||
@@ -340,7 +341,7 @@ public abstract class AbstractLegacyDataSourceHandler<TDataSource extends IDataS
|
||||
CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32());
|
||||
// normally a DhStream should be the topmost stream to prevent closing the stream accidentally,
|
||||
// but since this stream will be closed immediately after writing anyway, it won't be an issue
|
||||
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut);
|
||||
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut, EDhApiDataCompressionMode.LZ4);
|
||||
|
||||
dataSource.writeToStream(compressedOut, AbstractLegacyDataSourceHandler.this.level);
|
||||
|
||||
|
||||
+4
-1
@@ -19,6 +19,7 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.file.fullDatafile;
|
||||
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import com.seibel.distanthorizons.core.config.Config;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource;
|
||||
@@ -138,7 +139,9 @@ public class NewFullDataFileHandler
|
||||
{
|
||||
try
|
||||
{
|
||||
return NewFullDataSourceDTO.CreateFromDataSource(dataSource);
|
||||
// when creating new data use the compressor currently selected in the config
|
||||
EDhApiDataCompressionMode compressionModeEnum = Config.Client.Advanced.LodBuilding.dataCompression.get();
|
||||
return NewFullDataSourceDTO.CreateFromDataSource(dataSource, compressionModeEnum);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.sql.dto;
|
||||
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
|
||||
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
@@ -71,7 +72,7 @@ public class LegacyDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
public DhDataInputStream getInputStream() throws IOException
|
||||
{
|
||||
InputStream inputStream = new ByteArrayInputStream(this.dataArray);
|
||||
DhDataInputStream compressedStream = new DhDataInputStream(inputStream);
|
||||
DhDataInputStream compressedStream = new DhDataInputStream(inputStream, EDhApiDataCompressionMode.LZ4); // LZ4 was used by DH before 2.1.0 and as such must be used until the render data format is changed to record the compressor
|
||||
return compressedStream;
|
||||
}
|
||||
|
||||
|
||||
+20
-18
@@ -19,7 +19,9 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.sql.dto;
|
||||
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
|
||||
import com.seibel.distanthorizons.core.config.Config;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
@@ -44,16 +46,15 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
/** only for the data array */
|
||||
public int dataChecksum;
|
||||
|
||||
//public long[][] dataArray;
|
||||
public byte[] dataByteArray;
|
||||
|
||||
/** @see EDhApiWorldGenerationStep */
|
||||
public byte[] columnGenStepByteArray;
|
||||
|
||||
//public FullDataPointIdMap mapping;
|
||||
public byte[] mappingByteArray;
|
||||
|
||||
public byte dataFormatVersion;
|
||||
public EDhApiDataCompressionMode compressionModeEnum;
|
||||
|
||||
public boolean applyToParent;
|
||||
|
||||
@@ -66,14 +67,14 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
// constructor //
|
||||
//=============//
|
||||
|
||||
public static NewFullDataSourceDTO CreateFromDataSource(NewFullDataSource dataSource) throws IOException
|
||||
public static NewFullDataSourceDTO CreateFromDataSource(NewFullDataSource dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException
|
||||
{
|
||||
CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints);
|
||||
byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping());
|
||||
CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum);
|
||||
byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping(), compressionModeEnum);
|
||||
|
||||
return new NewFullDataSourceDTO(
|
||||
dataSource.getSectionPos(),
|
||||
checkedDataPointArray.checksum, dataSource.columnGenerationSteps, NewFullDataSource.DATA_FORMAT_VERSION, checkedDataPointArray.byteArray,
|
||||
checkedDataPointArray.checksum, dataSource.columnGenerationSteps, NewFullDataSource.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray,
|
||||
dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime,
|
||||
mappingByteArray, dataSource.applyToParent,
|
||||
dataSource.levelMinY
|
||||
@@ -81,7 +82,7 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
}
|
||||
public NewFullDataSourceDTO(
|
||||
DhSectionPos pos,
|
||||
int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, byte[] dataByteArray,
|
||||
int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] dataByteArray,
|
||||
long lastModifiedUnixDateTime, long createdUnixDateTime,
|
||||
byte[] mappingByteArray, boolean applyToParent,
|
||||
int levelMinY)
|
||||
@@ -91,6 +92,7 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
this.columnGenStepByteArray = columnGenStepByteArray;
|
||||
|
||||
this.dataFormatVersion = dataFormatVersion;
|
||||
this.compressionModeEnum = compressionModeEnum;
|
||||
|
||||
this.dataByteArray = dataByteArray;
|
||||
this.mappingByteArray = mappingByteArray;
|
||||
@@ -130,7 +132,7 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
}
|
||||
|
||||
dataSource.columnGenerationSteps = this.columnGenStepByteArray;
|
||||
dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray);
|
||||
dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray, this.compressionModeEnum);
|
||||
|
||||
dataSource.getMapping().clear(dataSource.getSectionPos());
|
||||
// should only be null when used in a unit test
|
||||
@@ -141,7 +143,7 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests.");
|
||||
}
|
||||
|
||||
dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper));
|
||||
dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper, this.compressionModeEnum));
|
||||
}
|
||||
|
||||
dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime;
|
||||
@@ -160,7 +162,7 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
// (de)serializing //
|
||||
//=================//
|
||||
|
||||
private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray) throws IOException
|
||||
private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
|
||||
{
|
||||
// write the outputs to a stream to prep for writing to the database
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
@@ -169,7 +171,7 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32());
|
||||
// normally a DhStream should be the topmost stream to prevent closing the stream accidentally,
|
||||
// but since this stream will be closed immediately after writing anyway, it won't be an issue
|
||||
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut);
|
||||
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut, compressionModeEnum);
|
||||
|
||||
|
||||
// write the data
|
||||
@@ -180,7 +182,7 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
|
||||
// write column length
|
||||
int columnLength = (dataColumn != null) ? dataColumn.length : 0;
|
||||
compressedOut.writeInt(columnLength); // TODO
|
||||
compressedOut.writeInt(columnLength); /// TODO
|
||||
|
||||
// write column data (will be skipped if no data was present)
|
||||
for (int y = 0; y < columnLength; y++)
|
||||
@@ -197,10 +199,10 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
|
||||
return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray());
|
||||
}
|
||||
private static long[][] readBlobToDataSourceDataArray(byte[] dataByteArray) throws IOException
|
||||
private static long[][] readBlobToDataSourceDataArray(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
|
||||
{
|
||||
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray);
|
||||
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream);
|
||||
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
|
||||
|
||||
|
||||
// read the data
|
||||
@@ -227,10 +229,10 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
}
|
||||
|
||||
|
||||
private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping) throws IOException
|
||||
private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException
|
||||
{
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream);
|
||||
DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum);
|
||||
|
||||
mapping.serialize(compressedOut);
|
||||
|
||||
@@ -239,10 +241,10 @@ public class NewFullDataSourceDTO implements IBaseDTO<DhSectionPos>
|
||||
|
||||
return byteArrayOutputStream.toByteArray();
|
||||
}
|
||||
private static FullDataPointIdMap readBlobToDataMapping(byte[] dataByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException
|
||||
private static FullDataPointIdMap readBlobToDataMapping(byte[] dataByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException
|
||||
{
|
||||
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray);
|
||||
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream);
|
||||
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
|
||||
|
||||
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper);
|
||||
return mapping;
|
||||
|
||||
+36
-11
@@ -19,6 +19,7 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.sql.repo;
|
||||
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO;
|
||||
|
||||
@@ -71,6 +72,8 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
|
||||
|
||||
byte dataFormatVersion = (Byte) objectMap.get("DataFormatVersion");
|
||||
byte compressionMode = (Byte) objectMap.get("CompressionMode");
|
||||
EDhApiDataCompressionMode compressionModeEnum = EDhApiDataCompressionMode.getFromValue(compressionMode);
|
||||
|
||||
boolean applyToParent = ((int) objectMap.get("ApplyToParent")) == 1;
|
||||
|
||||
@@ -79,7 +82,7 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
|
||||
NewFullDataSourceDTO dto = new NewFullDataSourceDTO(
|
||||
pos,
|
||||
dataChecksum, columnGenStepByteArray, dataFormatVersion, dataByteArray,
|
||||
dataChecksum, columnGenStepByteArray, dataFormatVersion, compressionModeEnum, dataByteArray,
|
||||
lastModifiedUnixDateTime, createdUnixDateTime,
|
||||
mappingByteArray, applyToParent,
|
||||
minY);
|
||||
@@ -94,13 +97,13 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
" DetailLevel, PosX, PosZ, \n" +
|
||||
" MinY, DataChecksum, \n" +
|
||||
" Data, ColumnGenerationStep, Mapping, \n" +
|
||||
" DataFormatVersion, ApplyToParent, \n" +
|
||||
" DataFormatVersion, CompressionMode, ApplyToParent, \n" +
|
||||
" LastModifiedUnixDateTime, CreatedUnixDateTime) \n" +
|
||||
"VALUES( \n" +
|
||||
" ?, ?, ?, \n" +
|
||||
" ?, ?, \n" +
|
||||
" ?, ?, ?, \n" +
|
||||
" ?, ?, \n" +
|
||||
" ?, ?, ?, \n" +
|
||||
" ?, ? \n" +
|
||||
");";
|
||||
PreparedStatement statement = this.createPreparedStatement(sql);
|
||||
@@ -118,6 +121,7 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
statement.setObject(i++, dto.mappingByteArray);
|
||||
|
||||
statement.setObject(i++, dto.dataFormatVersion);
|
||||
statement.setObject(i++, dto.compressionModeEnum.value);
|
||||
statement.setObject(i++, dto.applyToParent);
|
||||
|
||||
statement.setObject(i++, dto.lastModifiedUnixDateTime);
|
||||
@@ -140,6 +144,7 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
" ,Mapping = ? \n" +
|
||||
|
||||
" ,DataFormatVersion = ? \n" +
|
||||
" ,CompressionMode = ? \n" +
|
||||
" ,ApplyToParent = ? \n" +
|
||||
|
||||
" ,LastModifiedUnixDateTime = ? \n" +
|
||||
@@ -157,6 +162,7 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
statement.setObject(i++, dto.mappingByteArray);
|
||||
|
||||
statement.setObject(i++, dto.dataFormatVersion);
|
||||
statement.setObject(i++, dto.compressionModeEnum.value);
|
||||
statement.setObject(i++, dto.applyToParent);
|
||||
|
||||
statement.setObject(i++, dto.lastModifiedUnixDateTime);
|
||||
@@ -262,7 +268,7 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
* @return the size of the full data at the given position
|
||||
* (doesn't include the size of the mapping or any other column)
|
||||
*/
|
||||
public int getDataSizeInBytes(DhSectionPos pos)
|
||||
public long getDataSizeInBytes(DhSectionPos pos)
|
||||
{
|
||||
int detailLevel = pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
|
||||
|
||||
@@ -270,20 +276,39 @@ public class NewFullDataSourceRepo extends AbstractDhRepo<DhSectionPos, NewFullD
|
||||
"select LENGTH(Data) as dataSize " +
|
||||
"from "+this.getTableName()+" " +
|
||||
"WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ());
|
||||
|
||||
int dataLength = (resultMap != null) ? (int) resultMap.get("dataSize") : 0;
|
||||
return dataLength;
|
||||
|
||||
if (resultMap != null && resultMap.get("dataSize") != null)
|
||||
{
|
||||
// Number cast is necessary because the returned number can be an int or long
|
||||
Number resultNumber = (Number) resultMap.get("dataSize");
|
||||
long dataLength = resultNumber.longValue();
|
||||
return dataLength;
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** @return the total size in bytes of the full data for this entire database */
|
||||
public int getTotalDataSizeInBytes()
|
||||
public long getTotalDataSizeInBytes()
|
||||
{
|
||||
Map<String, Object> resultMap = this.queryDictionaryFirst(
|
||||
"select SUM(LENGTH(Data)) as dataSize " +
|
||||
"from "+this.getTableName()+"; ");
|
||||
|
||||
int dataLength = (resultMap != null) ? (int) resultMap.get("dataSize") : 0;
|
||||
return dataLength;
|
||||
|
||||
if (resultMap != null && resultMap.get("dataSize") != null)
|
||||
{
|
||||
Number resultNumber = (Number) resultMap.get("dataSize");
|
||||
long dataLength = resultNumber.longValue();
|
||||
return dataLength;
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
+24
-6
@@ -19,12 +19,12 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.util.objects.dataStreams;
|
||||
|
||||
import com.github.luben.zstd.ZstdInputStream;
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import net.jpountz.lz4.LZ4FrameInputStream;
|
||||
import org.tukaani.xz.XZInputStream;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.*;
|
||||
|
||||
/**
|
||||
* Combines multiple different streams together for ease of use
|
||||
@@ -38,9 +38,27 @@ import java.io.InputStream;
|
||||
*/
|
||||
public class DhDataInputStream extends DataInputStream
|
||||
{
|
||||
public DhDataInputStream(InputStream stream) throws IOException
|
||||
public DhDataInputStream(InputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException
|
||||
{
|
||||
super(warpStream(new BufferedInputStream(stream), compressionMode));
|
||||
}
|
||||
private static InputStream warpStream(InputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException
|
||||
{
|
||||
super(new LZ4FrameInputStream(new BufferedInputStream(stream)));
|
||||
switch (compressionMode)
|
||||
{
|
||||
case UNCOMPRESSED:
|
||||
return stream;
|
||||
case LZ4:
|
||||
return new LZ4FrameInputStream(stream);
|
||||
case Z_STD:
|
||||
return new ZstdInputStream(stream);
|
||||
case LZMA2:
|
||||
// Note: all LZMA/XZ compressors can be decompressed using this same InputStream
|
||||
return new XZInputStream(stream);
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("No compressor defined for ["+compressionMode+"]");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+25
-2
@@ -19,7 +19,11 @@
|
||||
|
||||
package com.seibel.distanthorizons.core.util.objects.dataStreams;
|
||||
|
||||
import com.github.luben.zstd.ZstdOutputStream;
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import net.jpountz.lz4.LZ4FrameOutputStream;
|
||||
import org.tukaani.xz.LZMA2Options;
|
||||
import org.tukaani.xz.XZOutputStream;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
@@ -30,9 +34,28 @@ import java.io.*;
|
||||
*/
|
||||
public class DhDataOutputStream extends DataOutputStream
|
||||
{
|
||||
public DhDataOutputStream(OutputStream stream) throws IOException
|
||||
public DhDataOutputStream(OutputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException
|
||||
{
|
||||
super(warpStream(new BufferedOutputStream(stream), compressionMode));
|
||||
}
|
||||
private static OutputStream warpStream(OutputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException
|
||||
{
|
||||
super(new LZ4FrameOutputStream(new BufferedOutputStream(stream)));
|
||||
switch (compressionMode)
|
||||
{
|
||||
case UNCOMPRESSED:
|
||||
return stream;
|
||||
case LZ4:
|
||||
return new LZ4FrameOutputStream(stream);
|
||||
case Z_STD:
|
||||
return new ZstdOutputStream(stream);
|
||||
case LZMA2:
|
||||
// in James' testing preset 4 has the best balance between compression ratio and speed
|
||||
// 5 is slightly more compressed 0.128 vs 0.139, but is roughly 60% slower
|
||||
return new XZOutputStream(stream, new LZMA2Options(4));
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("No compressor defined for ["+compressionMode+"]");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -341,13 +341,18 @@
|
||||
|
||||
"distanthorizons.config.client.advanced.lodBuilding.minTimeBetweenChunkUpdatesInSeconds":
|
||||
"Minimum Time Between Chunk Updates In Seconds",
|
||||
|
||||
"distanthorizons.config.client.advanced.lodBuilding.minTimeBetweenChunkUpdatesInSeconds.@tooltip":
|
||||
"Determines how long must pass between LOD chunk updates before another. \nupdate can occur\n\nIncreasing this value will reduce CPU load but may may cause \nLODs to become outdated more frequently or for longer.",
|
||||
"distanthorizons.config.client.advanced.lodBuilding.onlyUseDhLightingEngine":
|
||||
"Only Use DH Lighting Engine",
|
||||
"distanthorizons.config.client.advanced.lodBuilding.onlyUseDhLightingEngine.@tooltip":
|
||||
"If false LODs will be lit by Minecraft's lighting engine when possible \nand fall back to the DH lighting engine only when necessary. \n\nIf true LODs will only be lit using Distant Horizons' lighting engine. \n\nGenerally it is best to leave this disabled and should only be enabled \nif there are lighting issues or for debugging.",
|
||||
|
||||
"distanthorizons.config.client.advanced.lodBuilding.dataCompression":
|
||||
"Data Compression",
|
||||
"distanthorizons.config.client.advanced.lodBuilding.dataCompression.@tooltip":
|
||||
"What algorithm should be used to compress new LOD data? \nThis setting will only affect new or updated LOD data, \nany data already generated when this setting is changed will be \nunaffected until it needs to be re-written to the database. \n\nFastest: LZ4 \nHighest Compression: LZMA2",
|
||||
|
||||
|
||||
"distanthorizons.config.client.advanced.multiplayer":
|
||||
"Multiplayer",
|
||||
@@ -755,6 +760,15 @@
|
||||
"distanthorizons.config.enum.EDhApiDistantGeneratorMode.FULL":
|
||||
"Full",
|
||||
|
||||
"distanthorizons.config.enum.EDhApiDataCompressionMode.UNCOMPRESSED":
|
||||
"Uncompressed",
|
||||
"distanthorizons.config.enum.EDhApiDataCompressionMode.LZ4":
|
||||
"LZ4",
|
||||
"distanthorizons.config.enum.EDhApiDataCompressionMode.Z_STD":
|
||||
"Zstd",
|
||||
"distanthorizons.config.enum.EDhApiDataCompressionMode.LZMA2":
|
||||
"LZMA2",
|
||||
|
||||
"distanthorizons.config.enum.ELightGenerationMode.DISTANT_HORIZONS":
|
||||
"Distant Horizons",
|
||||
"distanthorizons.config.enum.ELightGenerationMode.MINECRAFT":
|
||||
|
||||
@@ -32,6 +32,7 @@ CREATE TABLE FullData (
|
||||
,Mapping BLOB NULL
|
||||
|
||||
,DataFormatVersion TINYINT NULL
|
||||
,CompressionMode TINYINT NULL
|
||||
|
||||
,ApplyToParent BIT NULL
|
||||
|
||||
|
||||
@@ -19,152 +19,212 @@
|
||||
|
||||
package tests;
|
||||
|
||||
import net.jpountz.lz4.*;
|
||||
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO;
|
||||
import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.text.CharacterIterator;
|
||||
import java.text.StringCharacterIterator;
|
||||
import java.util.ArrayList;
|
||||
|
||||
//import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
|
||||
//import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
|
||||
//import com.github.luben.zstd.ZstdInputStream;
|
||||
//import com.github.luben.zstd.ZstdOutputStream;
|
||||
|
||||
/**
|
||||
* Results (2023-5-20): <br>
|
||||
* 200 files <br><br>
|
||||
*
|
||||
* <strong>uncompressed</strong> <br><br>
|
||||
*
|
||||
* render data - ratio 1.0 (shocker :P) <br>
|
||||
* read time in - 784 ms, avg 3 ms/file <br>
|
||||
* write time in - 803 ms, avg 4 ms/file <br><br>
|
||||
*
|
||||
* full data - ratio 1.0 <br>
|
||||
* read time in - 2,213 ms, avg 11 ms/file <br>
|
||||
* write time in - 1,753 ms, avg 8 ms/file <br><br><br>
|
||||
*
|
||||
*
|
||||
* <strong>XZ</strong> <br><br>
|
||||
*
|
||||
* render data - ratio 0.1044 <br>
|
||||
* read time in - 2,413 ms, avg 12 ms/file <br>
|
||||
* write time in - 28,441 ms, avg 142 ms/file <br><br>
|
||||
*
|
||||
* full data - ratio 0.1123 <br>
|
||||
* read time in - 5,888 ms, avg 29 ms/file <br>
|
||||
* write time in - 79,675 ms, avg 398 ms/file <br><br><br>
|
||||
*
|
||||
*
|
||||
* <strong>LZ4</strong> <br><br>
|
||||
*
|
||||
* render data - ratio 0.2933 <br>
|
||||
* read time in - 846 ms, avg 4 ms/file <br>
|
||||
* write time in - 1,040 ms, avg 5 ms/file <br><br>
|
||||
*
|
||||
* full data - ratio 0.3275 <br>
|
||||
* read time in - 1,964 ms, avg 9 ms/file <br>
|
||||
* write time in - 1,584 ms, avg 7 ms/file <br><br><br>
|
||||
*
|
||||
*
|
||||
* <strong>Z Standard</strong> <br><br>
|
||||
*
|
||||
* render data - ratio 0.1791 <br>
|
||||
* read time in - 5,170 ms, avg 25 ms/file <br>
|
||||
* write time in - 5,294 ms, avg 26 ms/file <br><br>
|
||||
*
|
||||
* full data - ratio 0.2060 <br>
|
||||
* read time in - 14,754 ms, avg 73 ms/file <br>
|
||||
* write time in - 14,057 ms, avg 70 ms/file <br><br><br>
|
||||
*
|
||||
*
|
||||
*
|
||||
* <strong>Note:</strong>
|
||||
* In order to test the compressors that aren't currently in use: <br>
|
||||
* 1. Generate DH data and point the {@link CompressionTest#TEST_DIR} variable to the "Distant_Horizons" folder.
|
||||
* 2. Add the following to build.gradle's dependencies block: <br>
|
||||
* 1. Generate DH data (64 DH render distance is suggest)
|
||||
* 2. Point the {@link CompressionTest#TEST_DIR} variable to the world's "data" folder.
|
||||
* 3. (Optional) Add the following to build.gradle's dependencies block: <br>
|
||||
* <code>
|
||||
* shadowMe("org.tukaani:xz:1.9")
|
||||
* shadowMe("org.apache.commons:commons-compress:1.21")
|
||||
* shadowMe("com.github.luben:zstd-jni:1.5.5-3")
|
||||
* forgeShadowMe('org.apache.commons:commons-compress:1.26.1')
|
||||
* forgeShadowMe('org.itadaki:bzip2:0.9.1')
|
||||
* forgeShadowMe('lzma:lzma:0.0.1')
|
||||
* </code><br>
|
||||
* 3. Uncomment the tests in this file <br>
|
||||
* 4. Run the tests like normal
|
||||
* 4. (Optional) Uncomment the tests in this file <br>
|
||||
* 5. Run tests like normal
|
||||
*/
|
||||
public class CompressionTest
|
||||
{
|
||||
public static String TEST_DIR = "C:\\DistantHorizonsWorkspace\\distantHorizons\\fabric\\run\\saves\\Arcapelago\\data\\Distant_Horizons";
|
||||
public static String RENDER_DATA_PATH = TEST_DIR + "\\renderCache";
|
||||
public static String FULL_DATA_PATH = TEST_DIR + "\\data";
|
||||
public static String TEST_DIR = "C:\\DistantHorizonsWorkspace\\distant-horizons\\run\\saves\\Arcapelago\\data";
|
||||
public static String DB_FILE_NAME_PREFIX = "DistantHorizons";
|
||||
public static String UNCOMPRESSED_DB_FILE_NAME = "DistantHorizons.sqlite";
|
||||
|
||||
/** limits the number of files tested so I don't have to wait 10 minutes for the slower compressors */
|
||||
public static int MAX_NUMBER_OF_FILES_TO_TEST = 200;
|
||||
/** -1 will test all of them */
|
||||
public static int MAX_DTO_TEST_COUNT = -1;
|
||||
|
||||
|
||||
|
||||
// @Test
|
||||
//@Test
|
||||
public void NoCompression()
|
||||
{
|
||||
String compressorName = "Uncompressed";
|
||||
|
||||
CreateInputStreamFunc createInputStreamFunc = (inputStream) -> inputStream;
|
||||
CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> outputStream;
|
||||
|
||||
|
||||
System.out.println(compressorName + " testing render data");
|
||||
this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
System.out.println(compressorName + " testing full data");
|
||||
this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
this.testCompressor(compressorName, EDhApiDataCompressionMode.UNCOMPRESSED);
|
||||
}
|
||||
|
||||
// @Test
|
||||
public void Lz4()
|
||||
// collapse the following commented out code when looking at tests
|
||||
|
||||
//@Test
|
||||
//public void GZIP() // DNF
|
||||
//{
|
||||
// String compressorName = "GZIP";
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new GZIPInputStream(inputStream);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new GZIPOutputStream(outputStream);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
//@Test
|
||||
//public void BZip2() // DNF
|
||||
//{
|
||||
// String compressorName = "bzip2";
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new BZip2InputStream(inputStream, true);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new BZip2OutputStream(outputStream);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
//@Test
|
||||
//public void blockLz4() // DNF
|
||||
//{
|
||||
// String compressorName = "Block LZ4";
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new BlockLZ4CompressorInputStream(inputStream);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new BlockLZ4CompressorOutputStream(outputStream);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
//@Test
|
||||
//public void lzma() // DNF, doesn't support flushing
|
||||
//{
|
||||
// String compressorName = "lzma";
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new LZMA2InputStream(inputStream, LZMA2Options.DICT_SIZE_MIN);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new LZMAOutputStream(outputStream, new LZMA2Options(), -1);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
//@Test
|
||||
//public void deflate() // DNF
|
||||
//{
|
||||
// String compressorName = "deflate";
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new DeflateCompressorInputStream(inputStream);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new DeflateCompressorOutputStream(outputStream);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
//@Test
|
||||
//public void snappy() // DNF
|
||||
//{
|
||||
// String compressorName = "snappy";
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new SnappyCompressorInputStream(inputStream);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new SnappyCompressorOutputStream(outputStream, Long.MAX_VALUE);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
|
||||
//@Test
|
||||
//public void Zstd()
|
||||
//{
|
||||
// String compressorName = "Zstd";
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new ZstdInputStream(inputStream);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new ZstdOutputStream(outputStream);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
////@Test
|
||||
//public void ZstdDictionary() throws SQLException // isn't any better than normal Zstd
|
||||
//{
|
||||
// String compressorName = "ZstdDictionary";
|
||||
//
|
||||
// BufferPool pool = RecyclingBufferPool.INSTANCE;
|
||||
//
|
||||
//
|
||||
// // create the dictionary
|
||||
// byte[] dictionary;
|
||||
// {
|
||||
// String uncompressedDatabaseFilePath = TEST_DIR + "/" + UNCOMPRESSED_DB_FILE_NAME;
|
||||
// NewFullDataSourceRepo uncompressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", uncompressedDatabaseFilePath);
|
||||
// ArrayList<DhSectionPos> positionList = uncompressedRepo.getAllPositions();
|
||||
//
|
||||
// // sample size of 10 MB or less
|
||||
// // dictionary size of 64 KB (1 MB and 10 MB's both seemed to perform worse)
|
||||
// ZstdDictTrainer dictTrainer = new ZstdDictTrainer(10 * 1024 * 1024, 64 * 1024);
|
||||
//
|
||||
// for (int i = 0; i < positionList.size(); i++)
|
||||
// {
|
||||
// DhSectionPos pos = positionList.get(i);
|
||||
// NewFullDataSourceDTO uncompressedDto = uncompressedRepo.getByKey(pos);
|
||||
//
|
||||
// dictTrainer.addSample(uncompressedDto.dataByteArray);
|
||||
// }
|
||||
//
|
||||
// dictionary = dictTrainer.trainSamples();
|
||||
// }
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) ->
|
||||
// {
|
||||
// ZstdInputStream stream = new ZstdInputStream(inputStream, pool);
|
||||
// stream.setDict(dictionary);
|
||||
// return stream;
|
||||
// };
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) ->
|
||||
// {
|
||||
// ZstdOutputStream stream = new ZstdOutputStream(outputStream, pool);
|
||||
// stream.setDict(dictionary);
|
||||
// return stream;
|
||||
// };
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
//@Test
|
||||
//public void Lz4FastCompression() // DNF
|
||||
//{
|
||||
// String compressorName = "LZ4FastCompression";
|
||||
//
|
||||
// LZ4FastDecompressor fastCompressor = LZ4Factory.fastestInstance().fastDecompressor();
|
||||
//
|
||||
// DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new LZ4BlockInputStream(inputStream, fastCompressor);
|
||||
// DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new LZ4BlockOutputStream(outputStream);
|
||||
//
|
||||
// this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc);
|
||||
//}
|
||||
|
||||
//@Test
|
||||
public void Lz4() // fast, poor compression
|
||||
{
|
||||
String compressorName = "LZ4";
|
||||
|
||||
CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new LZ4FrameInputStream(inputStream);
|
||||
CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new LZ4FrameOutputStream(outputStream);
|
||||
|
||||
|
||||
System.out.println(compressorName + " testing render data");
|
||||
this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
System.out.println(compressorName + " testing full data");
|
||||
this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
this.testCompressor(compressorName, EDhApiDataCompressionMode.LZ4);
|
||||
}
|
||||
|
||||
//@Test
|
||||
public void Zstd() // middle of the road
|
||||
{
|
||||
String compressorName = "Zstd";
|
||||
this.testCompressor(compressorName, EDhApiDataCompressionMode.Z_STD);
|
||||
}
|
||||
|
||||
//@Test
|
||||
public void LZMA2() // very slow, very good compression though
|
||||
{
|
||||
String compressorName = "LZMA";
|
||||
this.testCompressor(compressorName, EDhApiDataCompressionMode.LZMA2);
|
||||
}
|
||||
|
||||
// @Test
|
||||
// public void Zstandard()
|
||||
// {
|
||||
// String compressorName = "Z_std";
|
||||
//
|
||||
// CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new ZstdInputStream(inputStream);
|
||||
// CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new ZstdOutputStream(outputStream);
|
||||
//
|
||||
//
|
||||
// System.out.println(compressorName+" testing render data");
|
||||
// this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
// System.out.println(compressorName+" testing full data");
|
||||
// this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
// }
|
||||
|
||||
// @Test
|
||||
// public void Xz()
|
||||
// {
|
||||
// String compressorName = "XZ";
|
||||
//
|
||||
// CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new XZCompressorInputStream(inputStream);
|
||||
// CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new XZCompressorOutputStream(outputStream);
|
||||
//
|
||||
//
|
||||
// System.out.println(compressorName+" testing render data");
|
||||
// this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
// System.out.println(compressorName+" testing full data");
|
||||
// this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc);
|
||||
// }
|
||||
|
||||
|
||||
|
||||
@@ -172,149 +232,139 @@ public class CompressionTest
|
||||
// testing methods //
|
||||
//=================//
|
||||
|
||||
@FunctionalInterface
|
||||
public interface CreateInputStreamFunc
|
||||
private void testCompressor(String compressorName, EDhApiDataCompressionMode compressionMode)
|
||||
{
|
||||
InputStream apply(InputStream inputStream) throws Exception;
|
||||
System.out.println("\n");
|
||||
System.out.println("Testing " + compressorName);
|
||||
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface CreateOutputStreamFunc
|
||||
{
|
||||
OutputStream apply(OutputStream outputStream) throws Exception;
|
||||
|
||||
}
|
||||
|
||||
private void testCompressor(
|
||||
String compressorName, String inputFolderPath,
|
||||
CreateInputStreamFunc createInputStreamFunc,
|
||||
CreateOutputStreamFunc createOutputStreamFunc)
|
||||
{
|
||||
long totalUncompressedFileSizeInBytes = 0;
|
||||
long totalCompressedFileSizeInBytes = 0;
|
||||
long minUncompressedDtoSizeInBytes = Long.MAX_VALUE;
|
||||
long maxUncompressedDtoSizeInBytes = 0;
|
||||
long avgUncompressedDtoSizeInBytes = 0;
|
||||
|
||||
long minCompressedDtoSizeInBytes = Long.MAX_VALUE;
|
||||
long maxCompressedDtoSizeInBytes = 0;
|
||||
long avgCompressedDtoSizeInBytes = 0;
|
||||
|
||||
long totalReadTimeInNano = 0;
|
||||
long totalWriteTimeInNano = 0;
|
||||
|
||||
|
||||
long totalUncompressedFileSizeInBytes;
|
||||
long totalCompressedFileSizeInBytes;
|
||||
|
||||
long totalReadTimeInMs = 0;
|
||||
long totalWriteTimeInMs = 0;
|
||||
|
||||
try
|
||||
{
|
||||
File inputFolder = new File(inputFolderPath);
|
||||
File[] inputFileArray = inputFolder.listFiles();
|
||||
Assert.assertNotNull(inputFileArray);
|
||||
String uncompressedDatabaseFilePath = TEST_DIR + "/" + UNCOMPRESSED_DB_FILE_NAME;
|
||||
File uncompressedDatabaseFile = new File(uncompressedDatabaseFilePath);
|
||||
Assert.assertTrue(uncompressedDatabaseFile.exists());
|
||||
|
||||
File compressedFolder = new File(inputFolderPath + "\\" + compressorName);
|
||||
compressedFolder.delete();
|
||||
compressedFolder.mkdirs();
|
||||
NewFullDataSourceRepo uncompressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", uncompressedDatabaseFilePath);
|
||||
|
||||
|
||||
int processedFileCount = 0;
|
||||
for (File inputFile : inputFileArray)
|
||||
String compressedDatabaseFilePath = TEST_DIR + "/output/" + DB_FILE_NAME_PREFIX + "_" + compressorName + ".sqlite";
|
||||
File compressedDatabaseFile = new File(compressedDatabaseFilePath);
|
||||
compressedDatabaseFile.mkdirs();
|
||||
compressedDatabaseFile.delete();
|
||||
Assert.assertTrue(!compressedDatabaseFile.exists());
|
||||
NewFullDataSourceRepo compressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", compressedDatabaseFilePath);
|
||||
|
||||
|
||||
|
||||
ArrayList<DhSectionPos> positionList = uncompressedRepo.getAllPositions();
|
||||
totalUncompressedFileSizeInBytes = uncompressedRepo.getTotalDataSizeInBytes();
|
||||
System.out.println("Found [" + positionList.size() + "] DTOs.");
|
||||
|
||||
long processedDtoCount = 0;
|
||||
int maxTestPosition = (MAX_DTO_TEST_COUNT == -1) ? positionList.size() : MAX_DTO_TEST_COUNT;
|
||||
for (int i = 0; i < maxTestPosition; i++)
|
||||
{
|
||||
if (inputFile.isDirectory())
|
||||
try
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// can be used to speed up the tests
|
||||
if (processedFileCount >= MAX_NUMBER_OF_FILES_TO_TEST)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// uncompressed file input //
|
||||
ArrayList<Byte> originalFileByteArray = new ArrayList<>();
|
||||
totalUncompressedFileSizeInBytes += Files.size(inputFile.toPath());
|
||||
|
||||
try (FileInputStream fileStream = new FileInputStream(inputFile);
|
||||
BufferedInputStream bufferedStream = new BufferedInputStream(fileStream);
|
||||
DataInputStream dataStream = new DataInputStream(bufferedStream))
|
||||
{
|
||||
try
|
||||
DhSectionPos pos = positionList.get(i);
|
||||
if (i % 20 == 0)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
byte nextByte = dataStream.readByte();
|
||||
originalFileByteArray.add(nextByte);
|
||||
}
|
||||
System.out.println(i + "/" + maxTestPosition);
|
||||
}
|
||||
catch (EOFException e)
|
||||
{ /* end of file reached */ }
|
||||
|
||||
|
||||
|
||||
// uncompressed input //
|
||||
|
||||
NewFullDataSourceDTO uncompressedDto = uncompressedRepo.getByKey(pos);
|
||||
Assert.assertEquals(uncompressedDto.compressionModeEnum, EDhApiDataCompressionMode.UNCOMPRESSED);
|
||||
NewFullDataSource uncompressedDataSource = uncompressedDto.createUnitTestDataSource();
|
||||
|
||||
long uncompressedDtoSize = uncompressedRepo.getDataSizeInBytes(pos);
|
||||
minUncompressedDtoSizeInBytes = Math.min(uncompressedDtoSize, minUncompressedDtoSizeInBytes);
|
||||
maxUncompressedDtoSizeInBytes = Math.max(uncompressedDtoSize, maxUncompressedDtoSizeInBytes);
|
||||
avgUncompressedDtoSizeInBytes += uncompressedDtoSize;
|
||||
|
||||
|
||||
|
||||
// compress file //
|
||||
|
||||
long startWriteNanoTime = System.nanoTime();
|
||||
|
||||
NewFullDataSourceDTO compressedDto = NewFullDataSourceDTO.CreateFromDataSource(uncompressedDataSource, compressionMode);
|
||||
compressedRepo.save(compressedDto);
|
||||
|
||||
long endWriteNanoTime = System.nanoTime();
|
||||
totalWriteTimeInNano += (endWriteNanoTime - startWriteNanoTime);
|
||||
|
||||
|
||||
long compressedDtoSize = compressedRepo.getDataSizeInBytes(pos);
|
||||
minCompressedDtoSizeInBytes = Math.min(compressedDtoSize, minCompressedDtoSizeInBytes);
|
||||
maxCompressedDtoSizeInBytes = Math.max(compressedDtoSize, maxCompressedDtoSizeInBytes);
|
||||
avgCompressedDtoSizeInBytes += compressedDtoSize;
|
||||
|
||||
|
||||
|
||||
// read compressed file //
|
||||
|
||||
long startReadNanoTime = System.nanoTime();
|
||||
|
||||
compressedDto = compressedRepo.getByKey(pos);
|
||||
NewFullDataSource compressedDataSource = compressedDto.createUnitTestDataSource();
|
||||
|
||||
long endReadMsTime = System.nanoTime();
|
||||
totalReadTimeInNano += (endReadMsTime - startReadNanoTime);
|
||||
|
||||
|
||||
processedDtoCount++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// compress file //
|
||||
long startWriteMsTime = System.currentTimeMillis();
|
||||
|
||||
File compressedFile = new File(inputFolderPath + "\\" + compressorName + "\\" + inputFile.getName());
|
||||
compressedFile.delete();
|
||||
compressedFile.createNewFile();
|
||||
|
||||
try (FileOutputStream fileStream = new FileOutputStream(compressedFile);
|
||||
BufferedOutputStream bufferedStream = new BufferedOutputStream(fileStream);
|
||||
OutputStream compressorStream = createOutputStreamFunc.apply(bufferedStream);
|
||||
DataOutputStream dataStream = new DataOutputStream(compressorStream))
|
||||
catch (Exception | Error e)
|
||||
{
|
||||
for (byte nextByte : originalFileByteArray)
|
||||
{
|
||||
dataStream.writeByte(nextByte);
|
||||
}
|
||||
e.printStackTrace();
|
||||
Assert.fail(e.getMessage());
|
||||
}
|
||||
|
||||
long endWriteMsTime = System.currentTimeMillis();
|
||||
totalWriteTimeInMs += (endWriteMsTime - startWriteMsTime);
|
||||
|
||||
totalCompressedFileSizeInBytes += Files.size(compressedFile.toPath());
|
||||
|
||||
|
||||
|
||||
// read compressed file //
|
||||
long startReadMsTime = System.currentTimeMillis();
|
||||
ArrayList<Byte> compressedFileByteArray = new ArrayList<>();
|
||||
|
||||
try (FileInputStream fileStream = new FileInputStream(compressedFile);
|
||||
BufferedInputStream bufferedStream = new BufferedInputStream(fileStream);
|
||||
InputStream compressorStream = createInputStreamFunc.apply(bufferedStream);
|
||||
DataInputStream dataStream = new DataInputStream(compressorStream))
|
||||
{
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
byte nextByte = dataStream.readByte();
|
||||
compressedFileByteArray.add(nextByte);
|
||||
}
|
||||
}
|
||||
catch (EOFException e)
|
||||
{ /* end of file reached */ }
|
||||
}
|
||||
|
||||
long endReadMsTime = System.currentTimeMillis();
|
||||
totalReadTimeInMs += (endReadMsTime - startReadMsTime);
|
||||
|
||||
|
||||
// confirm the file contents are the same
|
||||
Assert.assertEquals("byte array size mismatch", compressedFileByteArray.size(), originalFileByteArray.size());
|
||||
for (int i = 0; i < compressedFileByteArray.size(); i++)
|
||||
{
|
||||
Assert.assertEquals("array content mismatch at index [" + i + "]", compressedFileByteArray.get(i), originalFileByteArray.get(i));
|
||||
}
|
||||
|
||||
|
||||
processedFileCount++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
totalCompressedFileSizeInBytes = compressedRepo.getTotalDataSizeInBytes();
|
||||
|
||||
avgCompressedDtoSizeInBytes /= processedDtoCount;
|
||||
avgUncompressedDtoSizeInBytes /= processedDtoCount;
|
||||
|
||||
|
||||
double compressionRatio = (totalCompressedFileSizeInBytes / (double) totalUncompressedFileSizeInBytes);
|
||||
String compressionRatioString = compressionRatio + "";
|
||||
compressionRatioString = compressionRatioString.substring(0, Math.min(6, compressionRatioString.length()));
|
||||
|
||||
System.out.println("Uncompressed file size: [" + humanReadableByteCountSI(totalUncompressedFileSizeInBytes) + "] Compressed file size: [" + humanReadableByteCountSI(totalCompressedFileSizeInBytes) + "]. Compression ratio: [" + compressionRatioString + "].");
|
||||
System.out.println("Total read time in MS: [" + totalReadTimeInMs + "] Average read time per file: [" + (totalReadTimeInMs / processedFileCount) + "]");
|
||||
System.out.println("Total write time in MS: [" + totalWriteTimeInMs + "] Average write time per file: [" + (totalWriteTimeInMs / processedFileCount) + "]");
|
||||
|
||||
System.out.println("\n");
|
||||
System.out.println("Results: " + compressorName);
|
||||
System.out.println();
|
||||
System.out.println("Total uncompressed data: [" + humanReadableByteCountSI(totalUncompressedFileSizeInBytes) + "] Total compressed data: [" + humanReadableByteCountSI(totalCompressedFileSizeInBytes) + "]. Compression ratio: [" + compressionRatioString + "].");
|
||||
System.out.println("Min uncompressed data: [" + humanReadableByteCountSI(minUncompressedDtoSizeInBytes) + "] Min compressed data: [" + humanReadableByteCountSI(minCompressedDtoSizeInBytes) + "].");
|
||||
System.out.println("Max uncompressed data: [" + humanReadableByteCountSI(maxUncompressedDtoSizeInBytes) + "] Max compressed data: [" + humanReadableByteCountSI(maxCompressedDtoSizeInBytes) + "].");
|
||||
System.out.println("Avg uncompressed data: [" + humanReadableByteCountSI(avgUncompressedDtoSizeInBytes) + "] Avg compressed data: [" + humanReadableByteCountSI(avgCompressedDtoSizeInBytes) + "].");
|
||||
System.out.println();
|
||||
System.out.println("Total read time in MS: [" + totalReadTimeInNano / 1_000_000.0 + "] Average read time per dto: [" + (totalReadTimeInNano / processedDtoCount) / 1_000_000.0 + "]");
|
||||
System.out.println("Total write time in MS: [" + totalWriteTimeInNano / 1_000_000.0 + "] Average write time per dto: [" + (totalWriteTimeInNano / processedDtoCount) / 1_000_000.0 + "]");
|
||||
System.out.println();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user