From 920ca5ec40cdcef2f8b465694490b236c3675144 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 21 May 2023 17:35:33 -0500 Subject: [PATCH] Add XZ compression for full and render data files Any old files people may have will probably throw errors and need to be regenerated. --- .../fullData/FullDataPointIdMap.java | 15 ++-- .../loader/AbstractFullDataSourceLoader.java | 4 +- .../loader/CompleteFullDataSourceLoader.java | 8 +-- ...hDetailIncompleteFullDataSourceLoader.java | 8 +-- ...wDetailIncompleteFullDataSourceLoader.java | 8 +-- .../sources/CompleteFullDataSource.java | 67 +++++++----------- .../HighDetailIncompleteFullDataSource.java | 60 ++++++---------- .../LowDetailIncompleteFullDataSource.java | 68 +++++++------------ .../sources/interfaces/IFullDataSource.java | 13 ++-- .../interfaces/IStreamableFullDataSource.java | 34 +++++----- .../render/ColumnRenderLoader.java | 10 ++- .../render/ColumnRenderSource.java | 25 ++++--- .../file/fullDatafile/FullDataMetaFile.java | 14 ++-- .../AbstractMetaDataContainerFile.java | 10 +-- .../file/renderfile/RenderMetaDataFile.java | 10 +-- .../util/objects/DhUnclosableInputStream.java | 14 ---- .../objects/DhUnclosableOutputStream.java | 14 ---- .../dataStreams/DhDataInputStream.java | 30 ++++++++ .../dataStreams/DhDataOutputStream.java | 21 ++++++ 19 files changed, 196 insertions(+), 237 deletions(-) delete mode 100644 core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableInputStream.java delete mode 100644 core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableOutputStream.java create mode 100644 core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataInputStream.java create mode 100644 core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataOutputStream.java diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java index d9dee29fb..f0d530ce0 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java @@ -1,6 +1,7 @@ package com.seibel.lod.core.dataObjects.fullData; import com.seibel.lod.core.dependencyInjection.SingletonInjector; +import com.seibel.lod.core.util.objects.dataStreams.*; import com.seibel.lod.core.wrapperInterfaces.IWrapperFactory; import com.seibel.lod.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.lod.core.wrapperInterfaces.world.IBiomeWrapper; @@ -60,25 +61,23 @@ public class FullDataPointIdMap } /** Serializes all contained entries into the given stream, formatted in UTF */ - public void serialize(BufferedOutputStream bufferedOutputStream) throws IOException + public void serialize(DhDataOutputStream outputStream) throws IOException { - DataOutputStream dataStream = new DataOutputStream(bufferedOutputStream); // DO NOT CLOSE! It would close all related streams - dataStream.writeInt(this.entries.size()); + outputStream.writeInt(this.entries.size()); for (Entry entry : this.entries) { - dataStream.writeUTF(entry.serialize()); + outputStream.writeUTF(entry.serialize()); } } /** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */ - public static FullDataPointIdMap deserialize(BufferedInputStream bufferedInputStream) throws IOException, InterruptedException + public static FullDataPointIdMap deserialize(DhDataInputStream inputStream) throws IOException, InterruptedException { - DataInputStream dataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE! It would close all related streams - int entityCount = dataStream.readInt(); + int entityCount = inputStream.readInt(); FullDataPointIdMap newMap = new FullDataPointIdMap(); for (int i = 0; i < entityCount; i++) { - newMap.entries.add(Entry.deserialize(dataStream.readUTF())); + newMap.entries.add(Entry.deserialize(inputStream.readUTF())); } return newMap; } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java index 52c9e53c0..ad515e8a2 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java @@ -4,8 +4,8 @@ import com.google.common.collect.HashMultimap; import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; -import java.io.BufferedInputStream; import java.io.IOException; import java.util.*; @@ -57,7 +57,7 @@ public abstract class AbstractFullDataSourceLoader * Can return null if any of the requirements aren't met. * @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down */ - public abstract IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException; + public abstract IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java index 4be4d23cf..ecbd36c04 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java @@ -4,8 +4,8 @@ import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSour import com.seibel.lod.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; -import java.io.BufferedInputStream; import java.io.IOException; public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader @@ -16,12 +16,10 @@ public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader } @Override - public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { - //TODO: Add decompressor here - CompleteFullDataSource dataSource = CompleteFullDataSource.createEmpty(dataFile.pos); - dataSource.populateFromStream(dataFile, bufferedInputStream, level); + dataSource.populateFromStream(dataFile, inputStream, level); return dataSource; } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java index 919b72f7a..c6a72521d 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java @@ -4,8 +4,8 @@ import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSour import com.seibel.lod.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; -import java.io.BufferedInputStream; import java.io.IOException; public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader @@ -16,12 +16,10 @@ public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSo } @Override - public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { - //TODO: Add decompressor here - HighDetailIncompleteFullDataSource dataSource = HighDetailIncompleteFullDataSource.createEmpty(dataFile.pos); - dataSource.populateFromStream(dataFile, bufferedInputStream, level); + dataSource.populateFromStream(dataFile, inputStream, level); return dataSource; } } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java index da0dce92d..20d445d9d 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java @@ -4,8 +4,8 @@ import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSour import com.seibel.lod.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; -import java.io.BufferedInputStream; import java.io.IOException; public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader @@ -15,12 +15,10 @@ public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSou } @Override - public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { - //TODO: Add decompressor here - LowDetailIncompleteFullDataSource dataSource = LowDetailIncompleteFullDataSource.createEmpty(dataFile.pos); - dataSource.populateFromStream(dataFile, bufferedInputStream, level); + dataSource.populateFromStream(dataFile, inputStream, level); return dataSource; } } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 2130c9732..fe2cb44fa 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -12,6 +12,7 @@ import com.seibel.lod.core.pos.DhBlockPos2D; import com.seibel.lod.core.pos.DhLodPos; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.pos.DhSectionPos; +import com.seibel.lod.core.util.objects.dataStreams.*; import com.seibel.lod.coreapi.util.BitShiftUtil; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.FullDataPointUtil; @@ -74,41 +75,36 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu @Override - public void writeSourceSummaryInfo(IDhLevel level, BufferedOutputStream bufferedOutputStream) throws IOException + public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - dataOutputStream.writeInt(this.getDataDetailLevel()); - dataOutputStream.writeInt(this.width); - dataOutputStream.writeInt(level.getMinY()); - dataOutputStream.writeByte(this.worldGenStep.value); + outputStream.writeInt(this.getDataDetailLevel()); + outputStream.writeInt(this.width); + outputStream.writeInt(level.getMinY()); + outputStream.writeByte(this.worldGenStep.value); } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException - { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE - - - int dataDetail = dataInputStream.readInt(); + public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException + { + int dataDetail = inputStream.readInt(); if (dataDetail != dataFile.baseMetaData.dataLevel) { throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.baseMetaData.dataLevel)); } - int width = dataInputStream.readInt(); + int width = inputStream.readInt(); if (width != WIDTH) { throw new IOException(LodUtil.formatLog("Section width mismatch: "+width+" != "+ WIDTH +" (Currently only 1 section width is supported)")); } - int minY = dataInputStream.readInt(); + int minY = inputStream.readInt(); if (minY != level.getMinY()) { LOGGER.warn("Data minY mismatch: "+minY+" != "+level.getMinY()+". Will ignore data's y level"); } - byte worldGenByte = dataInputStream.readByte(); + byte worldGenByte = inputStream.readByte(); EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(worldGenByte); if (worldGenStep == null) { @@ -126,17 +122,14 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu @Override - public boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException + public boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - if (this.isEmpty()) { - dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); + outputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); return false; } - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); @@ -145,14 +138,14 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu { for (int z = 0; z < this.width; z++) { - dataOutputStream.writeInt(this.get(x, z).getSingleLength()); + outputStream.writeInt(this.get(x, z).getSingleLength()); } } // Data array content (only on non-empty columns) - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); for (int x = 0; x < this.width; x++) { for (int z = 0; z < this.width; z++) @@ -163,7 +156,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu long[] dataPointArray = columnAccessor.getRaw(); for (long dataPoint : dataPointArray) { - dataOutputStream.writeLong(dataPoint); + outputStream.writeLong(dataPoint); } } } @@ -173,12 +166,8 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return true; } @Override - public long[][] readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException + public long[][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream dataInputStream) throws IOException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE - - - // Data array length int dataPresentFlag = dataInputStream.readInt(); if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) @@ -237,28 +226,22 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu @Override - public void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException + public void writeIdMappings(DhDataOutputStream outputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(bufferedOutputStream); + outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + this.mapping.serialize(outputStream); } @Override - public FullDataPointIdMap readIdMappings(long[][] dataPoints, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException + public FullDataPointIdMap readIdMappings(long[][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // Don't close, this stream is handled outside this method - - - - int guardByte = dataInputStream.readInt(); + int guardByte = inputStream.readInt(); if (guardByte != IFullDataSource.DATA_GUARD_BYTE) { throw new IOException("Invalid data content end guard for ID mapping"); } - return FullDataPointIdMap.deserialize(bufferedInputStream); + return FullDataPointIdMap.deserialize(inputStream); } @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java index 16781facc..3e655f195 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java @@ -13,6 +13,7 @@ import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.pos.DhLodPos; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.util.objects.dataStreams.*; import com.seibel.lod.coreapi.util.BitShiftUtil; import com.seibel.lod.core.util.FullDataPointUtil; import com.seibel.lod.core.util.LodUtil; @@ -110,11 +111,8 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo @Override - public void writeSourceSummaryInfo(IDhLevel level, BufferedOutputStream bufferedOutputStream) throws IOException + public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream dataOutputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - dataOutputStream.writeShort(this.getDataDetailLevel()); dataOutputStream.writeShort(SPARSE_UNIT_DETAIL); dataOutputStream.writeInt(SECTION_SIZE); @@ -123,22 +121,19 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE - - LodUtil.assertTrue(dataFile.pos.sectionDetailLevel > SPARSE_UNIT_DETAIL); LodUtil.assertTrue(dataFile.pos.sectionDetailLevel <= MAX_SECTION_DETAIL); - int dataDetail = dataInputStream.readShort(); + int dataDetail = inputStream.readShort(); if(dataDetail != dataFile.baseMetaData.dataLevel) { throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.baseMetaData.dataLevel)); } // confirm that the detail level is correct - int sparseDetail = dataInputStream.readShort(); + int sparseDetail = inputStream.readShort(); if (sparseDetail != SPARSE_UNIT_DETAIL) { throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}", @@ -146,20 +141,20 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo } // confirm the scale of the data points is correct - int sectionSize = dataInputStream.readInt(); + int sectionSize = inputStream.readInt(); if (sectionSize != SECTION_SIZE) { throw new IOException(LodUtil.formatLog( "Section size mismatch: {} != {} (Currently only 1 section size is supported)", sectionSize, SECTION_SIZE)); } - int minY = dataInputStream.readInt(); + int minY = inputStream.readInt(); if (minY != level.getMinY()) { LOGGER.warn("Data minY mismatch: "+minY+" != "+level.getMinY()+". Will ignore data's y level"); } - EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(dataInputStream.readByte()); + EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte()); if (worldGenStep == null) { worldGenStep = EDhApiWorldGenerationStep.SURFACE; @@ -173,11 +168,8 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo @Override - public boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException + public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - if (this.isEmpty) { dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); @@ -238,12 +230,8 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo return true; } @Override - public long[][][] readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException + public long[][][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE - - - // calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize // TODO these values should be constant, should we still be calculating them like this? int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.sectionDetailLevel - SPARSE_UNIT_DETAIL); @@ -251,7 +239,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo // check if this file has any data - int dataPresentFlag = dataInputStream.readInt(); + int dataPresentFlag = inputStream.readInt(); if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) { // this file is empty @@ -265,7 +253,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo // get the number of columns (IE the bitSet from before) - int numberOfDataColumns = dataInputStream.readInt(); + int numberOfDataColumns = inputStream.readInt(); // validate the number of data columns int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent? if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns) @@ -276,7 +264,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo // read in the presence of each data column byte[] bytes = new byte[numberOfDataColumns]; - dataInputStream.readFully(bytes, 0, numberOfDataColumns); + inputStream.readFully(bytes, 0, numberOfDataColumns); BitSet dataArrayIndexHasData = BitSet.valueOf(bytes); @@ -286,7 +274,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo //====================// // (only on non-empty columns) - int dataArrayStartByte = dataInputStream.readInt(); + int dataArrayStartByte = inputStream.readInt(); // confirm the column data is starting if (dataArrayStartByte != IFullDataSource.DATA_GUARD_BYTE) { @@ -309,7 +297,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo for (int x = 0; x < dataColumn.length; x++) { // this should be zero if the column doesn't have any data - int dataColumnLength = dataInputStream.readInt(); + int dataColumnLength = inputStream.readInt(); dataColumn[x] = new long[dataColumnLength]; } @@ -321,7 +309,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo // read in the data columns for (int z = 0; z < dataColumn[x].length; z++) { - dataColumn[x][z] = dataInputStream.readLong(); + dataColumn[x][z] = inputStream.readLong(); } } } @@ -361,23 +349,17 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo @Override - public void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException + public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(bufferedOutputStream); + this.mapping.serialize(dataOutputStream); } @Override - public FullDataPointIdMap readIdMappings(long[][][] dataPoints, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException + public FullDataPointIdMap readIdMappings(long[][][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // Don't close, this stream is handled outside this method - - // mark the start of the ID data - int idMappingStartByte = dataInputStream.readInt(); + int idMappingStartByte = inputStream.readInt(); if (idMappingStartByte != DATA_GUARD_BYTE) { // the file format is incorrect @@ -385,7 +367,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo } // deserialize the ID data - return FullDataPointIdMap.deserialize(bufferedInputStream); + return FullDataPointIdMap.deserialize(inputStream); } @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java index 1281a67ce..2513c1474 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java @@ -13,6 +13,7 @@ import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.pos.DhLodPos; import com.seibel.lod.core.pos.DhSectionPos; +import com.seibel.lod.core.util.objects.dataStreams.*; import com.seibel.lod.coreapi.util.BitShiftUtil; import com.seibel.lod.core.util.FullDataPointUtil; import com.seibel.lod.core.util.LodUtil; @@ -88,42 +89,36 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp @Override - public void writeSourceSummaryInfo(IDhLevel level, BufferedOutputStream bufferedOutputStream) throws IOException + public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - - dataOutputStream.writeInt(this.getDataDetailLevel()); - dataOutputStream.writeInt(this.width); - dataOutputStream.writeInt(level.getMinY()); - dataOutputStream.writeByte(this.worldGenStep.value); + outputStream.writeInt(this.getDataDetailLevel()); + outputStream.writeInt(this.width); + outputStream.writeInt(level.getMinY()); + outputStream.writeByte(this.worldGenStep.value); } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE - - - int dataDetail = dataInputStream.readInt(); + int dataDetail = inputStream.readInt(); if(dataDetail != dataFile.baseMetaData.dataLevel) { throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.baseMetaData.dataLevel)); } - int width = dataInputStream.readInt(); + int width = inputStream.readInt(); if (width != WIDTH) { throw new IOException(LodUtil.formatLog("Section size mismatch: "+width+" != "+ WIDTH +" (Currently only 1 section size is supported)")); } - int minY = dataInputStream.readInt(); + int minY = inputStream.readInt(); if (minY != level.getMinY()) { LOGGER.warn("Data minY mismatch: "+minY+" != "+level.getMinY()+". Will ignore data's y level"); } - EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(dataInputStream.readByte()); + EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte()); if (worldGenStep == null) { worldGenStep = EDhApiWorldGenerationStep.SURFACE; @@ -140,11 +135,8 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp @Override - public boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException + public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - if (this.isEmpty) { dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); @@ -174,14 +166,10 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp return true; } @Override - public StreamDataPointContainer readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException + public StreamDataPointContainer readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE - - - // is source empty flag - int dataPresentFlag = dataInputStream.readInt(); + int dataPresentFlag = inputStream.readInt(); if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) { // Section is empty @@ -194,7 +182,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp // data column presence - int length = dataInputStream.readInt(); + int length = inputStream.readInt(); if (length < 0 || length > (WIDTH * WIDTH /8+64)*2) // TODO replace magic numbers or comment what they mean { throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})", @@ -202,14 +190,14 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp } byte[] bytes = new byte[length]; - dataInputStream.readFully(bytes, 0, length); + inputStream.readFully(bytes, 0, length); BitSet isColumnNotEmpty = BitSet.valueOf(bytes); // Data array content long[][] dataPointArray = new long[WIDTH * WIDTH][]; - dataPresentFlag = dataInputStream.readInt(); + dataPresentFlag = inputStream.readInt(); if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) { throw new IOException("invalid spotty flag end guard"); @@ -217,10 +205,10 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp for (int xz = isColumnNotEmpty.nextSetBit(0); xz >= 0; xz = isColumnNotEmpty.nextSetBit(xz + 1)) { - long[] array = new long[dataInputStream.readByte()]; + long[] array = new long[inputStream.readByte()]; for (int y = 0; y < array.length; y++) { - array[y] = dataInputStream.readLong(); + array[y] = inputStream.readLong(); } dataPointArray[xz] = array; } @@ -248,28 +236,22 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp @Override - public void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException + public void writeIdMappings(DhDataOutputStream outputStream) throws IOException { - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method - - - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(bufferedOutputStream); + outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + this.mapping.serialize(outputStream); } @Override - public FullDataPointIdMap readIdMappings(StreamDataPointContainer streamDataPointContainer, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException + public FullDataPointIdMap readIdMappings(StreamDataPointContainer streamDataPointContainer, DhDataInputStream inputStream) throws IOException, InterruptedException { - DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // Don't close, this stream is handled outside this method - - // Id mapping - int dataPresentFlag = dataInputStream.readInt(); + int dataPresentFlag = inputStream.readInt(); if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) { throw new IOException("invalid ID mapping end guard"); } - return FullDataPointIdMap.deserialize(bufferedInputStream); + return FullDataPointIdMap.deserialize(inputStream); } @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java index 5c3be199f..7de7a153d 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java @@ -9,9 +9,9 @@ import com.seibel.lod.core.dataObjects.render.ColumnRenderSource; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.pos.DhSectionPos; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; +import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.IOException; import java.util.ArrayList; @@ -73,16 +73,17 @@ public interface IFullDataSource // basic stream handling // //=======================// + // TODO make this blow up in IStreamableFullDataSource instead of the children /** * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. - * @see IStreamableFullDataSource#writeToStream(BufferedOutputStream, IDhLevel) + * @see IStreamableFullDataSource#writeToStream(DhDataOutputStream, IDhLevel) */ - void writeToStream(BufferedOutputStream bufferedOutputStream, IDhLevel level) throws IOException; + void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException; /** * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. - * @see IStreamableFullDataSource#populateFromStream(FullDataMetaFile, BufferedInputStream, IDhLevel) + * @see IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel) */ - void populateFromStream(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException; + void populateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java index fb425f9c3..491765ee2 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java @@ -5,9 +5,8 @@ import com.seibel.lod.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.lod.core.dataObjects.fullData.accessor.FullDataArrayAccessor; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; +import com.seibel.lod.core.util.objects.dataStreams.*; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.IOException; /** @@ -19,7 +18,7 @@ import java.io.IOException; * * @apiNote James would've preferred to have this as an abstract class, * however that is impossible. See the apiNote in - * {@link IStreamableFullDataSource#populateFromStream(FullDataMetaFile, BufferedInputStream, IDhLevel) populateFromStream} + * {@link IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel) populateFromStream} * for the full reasoning. * * @param defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}. @@ -40,13 +39,13 @@ public interface IStreamableFullDataSource * This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed. * * @throws IOException if the {@link FullDataMetaFile} isn't valid for this object. */ - SummaryDataType readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException; + SummaryDataType readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException; void setSourceSummaryData(SummaryDataType summaryData); /** @return true if any data points were present and written, false if this object was empty */ - boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException; + boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException; /** @return null if no data points were present */ - DataContainerType readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException; + DataContainerType readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException; void setDataPoints(DataContainerType dataPoints); - void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException; - FullDataPointIdMap readIdMappings(DataContainerType dataPoints, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException; + void writeIdMappings(DhDataOutputStream outputStream) throws IOException; + FullDataPointIdMap readIdMappings(DataContainerType dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException; void setIdMapping(FullDataPointIdMap mappings); diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java index f0077ad75..561bb0813 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java @@ -10,10 +10,9 @@ import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.file.renderfile.RenderMetaDataFile; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; -import java.io.BufferedInputStream; -import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -36,9 +35,8 @@ public class ColumnRenderLoader - public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException + public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException { - DataInputStream inputDataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE int dataFileVersion = dataFile.baseMetaData.binaryDataFormatVersion; switch (dataFileVersion) @@ -46,7 +44,7 @@ public class ColumnRenderLoader case 1: //LOGGER.info("loading render source "+dataFile.pos); - ParsedColumnData parsedColumnData = readDataV1(inputDataStream, level.getMinY()); + ParsedColumnData parsedColumnData = readDataV1(inputStream, level.getMinY()); if (parsedColumnData.isEmpty) { LOGGER.warn("Empty render file "+dataFile.pos); @@ -85,7 +83,7 @@ public class ColumnRenderLoader * * @throws IOException if there was an issue reading the stream */ - private static ParsedColumnData readDataV1(DataInputStream inputStream, int expectedYOffset) throws IOException + private static ParsedColumnData readDataV1(DhDataInputStream inputStream, int expectedYOffset) throws IOException { // TODO move into ColumnRenderSource diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderSource.java index 38cb73551..c2ca46944 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderSource.java @@ -3,6 +3,7 @@ package com.seibel.lod.core.dataObjects.render; import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.lod.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; import com.seibel.lod.core.dataObjects.transformers.FullDataToRenderDataTransformer; +import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.lod.coreapi.ModInfo; import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.lod.core.dataObjects.render.columnViews.ColumnArrayView; @@ -11,7 +12,6 @@ import com.seibel.lod.core.dataObjects.render.columnViews.IColumnDataView; import com.seibel.lod.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer; import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.pos.DhSectionPos; -import com.seibel.lod.core.enums.ELodDirection; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.coreapi.util.BitShiftUtil; @@ -203,24 +203,23 @@ public class ColumnRenderSource // data update and output // //========================// - public void writeData(BufferedOutputStream bufferedOutputStream) throws IOException + public void writeData(DhDataOutputStream outputStream) throws IOException { - bufferedOutputStream.flush(); - DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); + outputStream.flush(); - dataOutputStream.writeByte(this.getDataDetail()); - dataOutputStream.writeInt(this.verticalDataCount); + outputStream.writeByte(this.getDataDetail()); + outputStream.writeInt(this.verticalDataCount); if (this.isEmpty) { // no data is present - dataOutputStream.writeByte(NO_DATA_FLAG_BYTE); + outputStream.writeByte(NO_DATA_FLAG_BYTE); } else { // data is present - dataOutputStream.writeByte(DATA_GUARD_BYTE); - dataOutputStream.writeInt(this.yOffset); + outputStream.writeByte(DATA_GUARD_BYTE); + outputStream.writeInt(this.yOffset); // write the data for each column for (int xz = 0; xz < SECTION_SIZE * SECTION_SIZE; xz++) @@ -228,15 +227,15 @@ public class ColumnRenderSource for (int y = 0; y < this.verticalDataCount; y++) { long currentDatapoint = this.renderDataContainer[xz * this.verticalDataCount + y]; - dataOutputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly + outputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly } } } - dataOutputStream.writeByte(DATA_GUARD_BYTE); - dataOutputStream.writeByte(this.worldGenStep.value); + outputStream.writeByte(DATA_GUARD_BYTE); + outputStream.writeByte(this.worldGenStep.value); - bufferedOutputStream.flush(); + outputStream.flush(); } /** Overrides any data that has not been written directly using write(). Skips empty source dataPoints. */ diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java index d0b324ab2..438792977 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java @@ -20,6 +20,7 @@ import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.AtomicsUtil; import com.seibel.lod.core.util.LodUtil; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; /** @@ -201,16 +202,16 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile { // Load the file. IFullDataSource fullDataSource; - try (FileInputStream fileInputStream = this._getFileInputStream(); - BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) + try (FileInputStream fileInputStream = this.getFileInputStream(); + DhDataInputStream compressedStream = new DhDataInputStream(fileInputStream)) { - fullDataSource = this.fullDataSourceLoader.loadData(this, bufferedInputStream, this.level); + fullDataSource = this.fullDataSourceLoader.loadData(this, compressedStream, this.level); } catch (Exception ex) { if (ex instanceof InterruptedException) { -// LOGGER.warn(FullDataMetaFile.class.getSimpleName()+" loadOrGetCachedAsync interrupted."); + //LOGGER.warn(FullDataMetaFile.class.getSimpleName()+" loadOrGetCachedAsync interrupted."); return null; } @@ -218,7 +219,6 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile throw new CompletionException(ex); } - // confirm that this thread is in control LodUtil.assertTrue(this.inCacheWriteAccessFuture.get() == null, "No one should be writing to the cache while we are in the process of " + @@ -263,7 +263,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile return future; } /** @return a stream for the data contained in this file, skips the metadata from {@link AbstractMetaDataContainerFile}. */ - private FileInputStream _getFileInputStream() throws IOException + private FileInputStream getFileInputStream() throws IOException { FileInputStream fileInputStream = new FileInputStream(this.file); @@ -456,7 +456,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile this.baseMetaData.dataTypeId = (this.fullDataSourceLoader == null) ? 0 : this.fullDataSourceLoader.datatypeId; this.baseMetaData.binaryDataFormatVersion = fullDataSource.getBinaryDataFormatVersion(); - super.writeData((outputStream) -> fullDataSource.writeToStream(outputStream, this.level)); + super.writeData((bufferedOutputStream) -> fullDataSource.writeToStream((bufferedOutputStream), this.level)); this.doesFileExist = true; } catch (ClosedByInterruptException e) // thrown by buffers that are interrupted diff --git a/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java b/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java index 20ed50bdd..dbee26bb8 100644 --- a/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java @@ -15,9 +15,9 @@ import java.util.zip.CheckedOutputStream; import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.lod.core.util.FileUtil; import com.seibel.lod.core.pos.DhSectionPos; -import com.seibel.lod.core.util.objects.DhUnclosableOutputStream; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; +import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream; import org.apache.logging.log4j.Logger; /** @@ -188,7 +188,7 @@ public abstract class AbstractMetaDataContainerFile } } - protected void writeData(IMetaDataWriterFunc dataWriterFunc) throws IOException + protected void writeData(IMetaDataWriterFunc dataWriterFunc) throws IOException { LodUtil.assertTrue(this.baseMetaData != null); if (this.file.exists()) @@ -212,10 +212,10 @@ public abstract class AbstractMetaDataContainerFile fileChannel.position(METADATA_SIZE_IN_BYTES); int checksum; - try(DhUnclosableOutputStream bufferedOut = new DhUnclosableOutputStream(Channels.newOutputStream(fileChannel)); // Prevent closing the channel by anything but closing the file channel - CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) // TODO: Is Adler32 ok? + try(DhDataOutputStream compressedOut = new DhDataOutputStream(Channels.newOutputStream(fileChannel)); + CheckedOutputStream checkedOut = new CheckedOutputStream(compressedOut, new Adler32())) // TODO: Is Adler32 ok? { - dataWriterFunc.writeBufferToFile(bufferedOut); // TODO it might be nice to have a DH stream we pass in instead of the base BufferedOutputStream to make it clear the streams can't be closed + dataWriterFunc.writeBufferToFile(compressedOut); checksum = (int) checkedOut.getChecksum().getValue(); } diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java index 33c711f82..3073b98b8 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java @@ -11,9 +11,9 @@ import com.seibel.lod.core.file.metaData.AbstractMetaDataContainerFile; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; +import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; -import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -177,10 +177,10 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile // Load the file. ColumnRenderSource renderSource; - try (FileInputStream fileInputStream = this.getDataContent(); - BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) + try (FileInputStream fileInputStream = this.getFileInputStream(); + DhDataInputStream compressedStream = new DhDataInputStream(fileInputStream)) { - renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, bufferedInputStream, level); + renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, compressedStream, level); } catch (IOException ex) { @@ -217,7 +217,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile renderSource.getDataDetail(), renderSource.worldGenStep, RenderSourceFileHandler.RENDER_SOURCE_TYPE_ID, renderSource.getRenderDataFormatVersion()); } - private FileInputStream getDataContent() throws IOException + private FileInputStream getFileInputStream() throws IOException { FileInputStream fin = new FileInputStream(this.file); int toSkip = METADATA_SIZE_IN_BYTES; diff --git a/core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableInputStream.java b/core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableInputStream.java deleted file mode 100644 index 5c2bf165c..000000000 --- a/core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableInputStream.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.seibel.lod.core.util.objects; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; - -public class DhUnclosableInputStream extends BufferedInputStream -{ - public DhUnclosableInputStream(InputStream it) { super(it); } - - @Override - public void close() throws IOException { /* Do nothing. */ } - -} diff --git a/core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableOutputStream.java b/core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableOutputStream.java deleted file mode 100644 index 6c59e62ec..000000000 --- a/core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableOutputStream.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.seibel.lod.core.util.objects; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -public class DhUnclosableOutputStream extends BufferedOutputStream -{ - public DhUnclosableOutputStream(OutputStream outputStream) { super(outputStream); } - - @Override - public void close() throws IOException { /* Do nothing. */ } - -} diff --git a/core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataInputStream.java b/core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataInputStream.java new file mode 100644 index 000000000..753b40afc --- /dev/null +++ b/core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataInputStream.java @@ -0,0 +1,30 @@ +package com.seibel.lod.core.util.objects.dataStreams; + +import net.jpountz.lz4.LZ4FrameInputStream; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Combines multiple different streams together for ease of use + * and to prevent accidentally wrapping a stream twice or passing in + * the wrong stream.

+ * + * Note: + * This stream cannot be closed, + * the passed in stream must be closed instead. + * This is done to prevent closing file channels prematurely while accessing them. + */ +public class DhDataInputStream extends DataInputStream +{ + public DhDataInputStream(InputStream stream) throws IOException + { + super(new LZ4FrameInputStream(new BufferedInputStream(stream))); + } + + @Override + public void close() throws IOException { /* Do nothing. */ } + +} diff --git a/core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataOutputStream.java b/core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataOutputStream.java new file mode 100644 index 000000000..1cc3f39ef --- /dev/null +++ b/core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataOutputStream.java @@ -0,0 +1,21 @@ +package com.seibel.lod.core.util.objects.dataStreams; + +import net.jpountz.lz4.LZ4FrameOutputStream; + +import java.io.*; + +/** + * See {@link DhDataInputStream} for more information about these custom streams. + * @see DhDataInputStream + */ +public class DhDataOutputStream extends DataOutputStream +{ + public DhDataOutputStream(OutputStream stream) throws IOException + { + super(new LZ4FrameOutputStream(new BufferedOutputStream(stream))); + } + + @Override + public void close() throws IOException { /* Do nothing. */ } + +}