From b38827a8de6a8c2ccf244f6faa8544889413e8dd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 May 2023 11:44:28 -0500 Subject: [PATCH] Add compression unit tests with results --- core/src/test/java/tests/CompressionTest.java | 334 ++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 core/src/test/java/tests/CompressionTest.java diff --git a/core/src/test/java/tests/CompressionTest.java b/core/src/test/java/tests/CompressionTest.java new file mode 100644 index 000000000..d84cfb8bb --- /dev/null +++ b/core/src/test/java/tests/CompressionTest.java @@ -0,0 +1,334 @@ +/* + * This file is part of the Distant Horizons mod (formerly the LOD Mod), + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2022 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import net.jpountz.lz4.*; +import org.junit.Assert; +import org.junit.Test; + +import java.io.*; +import java.nio.file.Files; +import java.text.CharacterIterator; +import java.text.StringCharacterIterator; +import java.util.ArrayList; + +//import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +//import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +//import com.github.luben.zstd.ZstdInputStream; +//import com.github.luben.zstd.ZstdOutputStream; + +/** + * Results (2023-5-20):
+ * 200 files

+ * + * uncompressed

+ * + * render data - ratio 1.0 (shocker :P)
+ * read time in - 784 ms, avg 3 ms/file
+ * write time in - 803 ms, avg 4 ms/file

+ * + * full data - ratio 1.0
+ * read time in - 2,213 ms, avg 11 ms/file
+ * write time in - 1,753 ms, avg 8 ms/file


+ * + * + * XZ

+ * + * render data - ratio 0.1044
+ * read time in - 2,413 ms, avg 12 ms/file
+ * write time in - 28,441 ms, avg 142 ms/file

+ * + * full data - ratio 0.1123
+ * read time in - 5,888 ms, avg 29 ms/file
+ * write time in - 79,675 ms, avg 398 ms/file


+ * + * + * LZ4

+ * + * render data - ratio 0.2933
+ * read time in - 846 ms, avg 4 ms/file
+ * write time in - 1,040 ms, avg 5 ms/file

+ * + * full data - ratio 0.3275
+ * read time in - 1,964 ms, avg 9 ms/file
+ * write time in - 1,584 ms, avg 7 ms/file


+ * + * + * Z Standard

+ * + * render data - ratio 0.1791
+ * read time in - 5,170 ms, avg 25 ms/file
+ * write time in - 5,294 ms, avg 26 ms/file

+ * + * full data - ratio 0.2060
+ * read time in - 14,754 ms, avg 73 ms/file
+ * write time in - 14,057 ms, avg 70 ms/file


+ * + * + * + * Note: + * In order to test the compressors that aren't currently in use:
+ * 1. uncomment the tests in this file
+ * 2. add the following to build.gradle's dependencies block:
+ * + * shadowMe("org.tukaani:xz:1.9") + * shadowMe("org.apache.commons:commons-compress:1.21") + * shadowMe("com.github.luben:zstd-jni:1.5.5-3") + * + * + */ +public class CompressionTest +{ + public static String TEST_DIR = "C:\\DistantHorizonsWorkspace\\distantHorizons\\fabric\\run\\saves\\Arcapelago\\data\\Distant_Horizons"; + public static String RENDER_DATA_PATH = TEST_DIR + "\\renderCache"; + public static String FULL_DATA_PATH = TEST_DIR + "\\data"; + + /** limits the number of files tested so I don't have to wait 10 minutes for the slower compressors */ + public static int MAX_NUMBER_OF_FILES_TO_TEST = 200; + + + + @Test + public void NoCompression() + { + String compressorName = "Uncompressed"; + + CreateInputStreamFunc createInputStreamFunc = (inputStream) -> inputStream; + CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> outputStream; + + + System.out.println(compressorName+" testing render data"); + this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); + System.out.println(compressorName+" testing full data"); + this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); + } + + @Test + public void Lz4() + { + String compressorName = "LZ4"; + + CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new LZ4FrameInputStream(inputStream); + CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new LZ4FrameOutputStream(outputStream); + + + System.out.println(compressorName+" testing render data"); + this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); + System.out.println(compressorName+" testing full data"); + this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); + } + +// @Test +// public void Zstandard() +// { +// String compressorName = "Z_std"; +// +// CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new ZstdInputStream(inputStream); +// CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new ZstdOutputStream(outputStream); +// +// +// System.out.println(compressorName+" testing render data"); +// this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); +// System.out.println(compressorName+" testing full data"); +// this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); +// } + +// @Test +// public void Xz() +// { +// String compressorName = "XZ"; +// +// CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new XZCompressorInputStream(inputStream); +// CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new XZCompressorOutputStream(outputStream); +// +// +// System.out.println(compressorName+" testing render data"); +// this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); +// System.out.println(compressorName+" testing full data"); +// this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); +// } + + + + //=================// + // testing methods // + //=================// + + @FunctionalInterface + public interface CreateInputStreamFunc { InputStream apply(InputStream inputStream) throws Exception; } + @FunctionalInterface + public interface CreateOutputStreamFunc { OutputStream apply(OutputStream outputStream) throws Exception; } + + private void testCompressor( + String compressorName, String inputFolderPath, + CreateInputStreamFunc createInputStreamFunc, + CreateOutputStreamFunc createOutputStreamFunc) + { + long totalUncompressedFileSizeInBytes = 0; + long totalCompressedFileSizeInBytes = 0; + + long totalReadTimeInMs = 0; + long totalWriteTimeInMs = 0; + + try + { + File inputFolder = new File(inputFolderPath); + File[] inputFileArray = inputFolder.listFiles(); + Assert.assertNotNull(inputFileArray); + + File compressedFolder = new File(inputFolderPath+"\\"+compressorName); + compressedFolder.delete(); + compressedFolder.mkdirs(); + + + int processedFileCount = 0; + for (File inputFile : inputFileArray) + { + if (inputFile.isDirectory()) + { + continue; + } + + // can be used to speed up the tests + if (processedFileCount >= MAX_NUMBER_OF_FILES_TO_TEST) + { + break; + } + + + + // uncompressed file input // + ArrayList originalFileByteArray = new ArrayList<>(); + totalUncompressedFileSizeInBytes += Files.size(inputFile.toPath()); + + try (FileInputStream fileStream = new FileInputStream(inputFile); + BufferedInputStream bufferedStream = new BufferedInputStream(fileStream); + DataInputStream dataStream = new DataInputStream(bufferedStream)) + { + try + { + while (true) + { + byte nextByte = dataStream.readByte(); + originalFileByteArray.add(nextByte); + } + } + catch (EOFException e) { /* end of file reached */ } + } + + + + // compress file // + long startWriteMsTime = System.currentTimeMillis(); + + File compressedFile = new File(inputFolderPath+"\\"+compressorName+"\\"+inputFile.getName()); + compressedFile.delete(); + compressedFile.createNewFile(); + + try (FileOutputStream fileStream = new FileOutputStream(compressedFile); + BufferedOutputStream bufferedStream = new BufferedOutputStream(fileStream); + OutputStream compressorStream = createOutputStreamFunc.apply(bufferedStream); + DataOutputStream dataStream = new DataOutputStream(compressorStream)) + { + for (byte nextByte : originalFileByteArray) + { + dataStream.writeByte(nextByte); + } + } + + long endWriteMsTime = System.currentTimeMillis(); + totalWriteTimeInMs += (endWriteMsTime - startWriteMsTime); + + totalCompressedFileSizeInBytes += Files.size(compressedFile.toPath()); + + + + // read compressed file // + long startReadMsTime = System.currentTimeMillis(); + ArrayList compressedFileByteArray = new ArrayList<>(); + + try (FileInputStream fileStream = new FileInputStream(compressedFile); + BufferedInputStream bufferedStream = new BufferedInputStream(fileStream); + InputStream compressorStream = createInputStreamFunc.apply(bufferedStream); + DataInputStream dataStream = new DataInputStream(compressorStream)) + { + try + { + while (true) + { + byte nextByte = dataStream.readByte(); + compressedFileByteArray.add(nextByte); + } + } + catch (EOFException e) { /* end of file reached */ } + } + + long endReadMsTime = System.currentTimeMillis(); + totalReadTimeInMs += (endReadMsTime - startReadMsTime); + + + // confirm the file contents are the same + Assert.assertEquals("byte array size mismatch", compressedFileByteArray.size(), originalFileByteArray.size()); + for (int i = 0; i < compressedFileByteArray.size(); i++) + { + Assert.assertEquals("array content mismatch at index ["+i+"]", compressedFileByteArray.get(i), originalFileByteArray.get(i)); + } + + + processedFileCount++; + } + + + double compressionRatio = (totalCompressedFileSizeInBytes / (double) totalUncompressedFileSizeInBytes); + String compressionRatioString = compressionRatio+""; + compressionRatioString = compressionRatioString.substring(0, Math.min(6,compressionRatioString.length())); + + System.out.println("Uncompressed file size: ["+humanReadableByteCountSI(totalUncompressedFileSizeInBytes)+"] Compressed file size: ["+humanReadableByteCountSI(totalCompressedFileSizeInBytes)+"]. Compression ratio: ["+compressionRatioString+"]."); + System.out.println("Total read time in MS: ["+totalReadTimeInMs+"] Average read time per file: ["+(totalReadTimeInMs/processedFileCount)+"]"); + System.out.println("Total write time in MS: ["+totalWriteTimeInMs+"] Average write time per file: ["+(totalWriteTimeInMs/processedFileCount)+"]"); + } + catch (Exception e) + { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + + /** + * Source: + * https://stackoverflow.com/questions/3758606/how-can-i-convert-byte-size-into-a-human-readable-format-in-java#3758880 + */ + public static String humanReadableByteCountSI(long bytes) + { + if (-1000 < bytes && bytes < 1000) + { + return bytes + " B"; + } + CharacterIterator ci = new StringCharacterIterator("kMGTPE"); + while (bytes <= -999_950 || bytes >= 999_950) + { + bytes /= 1000; + ci.next(); + } + return String.format("%.1f %cB", bytes / 1000.0, ci.current()); + } + +}