wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add XZ compression for full and render data files

Browse Source 
Any old files people may have will probably throw errors and need to be regenerated.
This commit is contained in:
James Seibel
2023-05-21 17:35:33 -05:00
parent 5986069eb9
commit 920ca5ec40
19 changed files with 196 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java
+7 -8
View File
@@ -1,6 +1,7 @@
package com.seibel.lod.core.dataObjects.fullData;
import com.seibel.lod.core.dependencyInjection.SingletonInjector;
import com.seibel.lod.core.util.objects.dataStreams.*;
import com.seibel.lod.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.lod.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.lod.core.wrapperInterfaces.world.IBiomeWrapper;
@@ -60,25 +61,23 @@ public class FullDataPointIdMap
}
/** Serializes all contained entries into the given stream, formatted in UTF */
public void serialize(BufferedOutputStream bufferedOutputStream) throws IOException
public void serialize(DhDataOutputStream outputStream) throws IOException
{
DataOutputStream dataStream = new DataOutputStream(bufferedOutputStream); // DO NOT CLOSE! It would close all related streams
dataStream.writeInt(this.entries.size());
outputStream.writeInt(this.entries.size());
for (Entry entry : this.entries)
{
dataStream.writeUTF(entry.serialize());
outputStream.writeUTF(entry.serialize());
}
}
/** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */
public static FullDataPointIdMap deserialize(BufferedInputStream bufferedInputStream) throws IOException, InterruptedException
public static FullDataPointIdMap deserialize(DhDataInputStream inputStream) throws IOException, InterruptedException
{
DataInputStream dataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE! It would close all related streams
int entityCount = dataStream.readInt();
int entityCount = inputStream.readInt();
FullDataPointIdMap newMap = new FullDataPointIdMap();
for (int i = 0; i < entityCount; i++)
{
newMap.entries.add(Entry.deserialize(dataStream.readUTF()));
newMap.entries.add(Entry.deserialize(inputStream.readUTF()));
}
return newMap;
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java
+2 -2
View File
@@ -4,8 +4,8 @@ import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.util.*;
@@ -57,7 +57,7 @@ public abstract class AbstractFullDataSourceLoader
* Can return null if any of the requirements aren't met.
* @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down
*/
public abstract IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException;
public abstract IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java
+3 -5
View File
@@ -4,8 +4,8 @@ import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSour
import com.seibel.lod.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import java.io.BufferedInputStream;
import java.io.IOException;
public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader
@@ -16,12 +16,10 @@ public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader
}
@Override
public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
//TODO: Add decompressor here
CompleteFullDataSource dataSource = CompleteFullDataSource.createEmpty(dataFile.pos);
dataSource.populateFromStream(dataFile, bufferedInputStream, level);
dataSource.populateFromStream(dataFile, inputStream, level);
return dataSource;
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java
+3 -5
View File
@@ -4,8 +4,8 @@ import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSour
import com.seibel.lod.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import java.io.BufferedInputStream;
import java.io.IOException;
public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader
@@ -16,12 +16,10 @@ public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSo
}
@Override
public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
//TODO: Add decompressor here
HighDetailIncompleteFullDataSource dataSource = HighDetailIncompleteFullDataSource.createEmpty(dataFile.pos);
dataSource.populateFromStream(dataFile, bufferedInputStream, level);
dataSource.populateFromStream(dataFile, inputStream, level);
return dataSource;
}
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java
+3 -5
View File
@@ -4,8 +4,8 @@ import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSour
import com.seibel.lod.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import java.io.BufferedInputStream;
import java.io.IOException;
public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader
@@ -15,12 +15,10 @@ public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSou
}
@Override
public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadData(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
//TODO: Add decompressor here
LowDetailIncompleteFullDataSource dataSource = LowDetailIncompleteFullDataSource.createEmpty(dataFile.pos);
dataSource.populateFromStream(dataFile, bufferedInputStream, level);
dataSource.populateFromStream(dataFile, inputStream, level);
return dataSource;
}
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java
+25 -42
View File
@@ -12,6 +12,7 @@ import com.seibel.lod.core.pos.DhBlockPos2D;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.objects.dataStreams.*;
import com.seibel.lod.coreapi.util.BitShiftUtil;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.FullDataPointUtil;
@@ -74,41 +75,36 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
@Override
public void writeSourceSummaryInfo(IDhLevel level, BufferedOutputStream bufferedOutputStream) throws IOException
public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
dataOutputStream.writeInt(this.getDataDetailLevel());
dataOutputStream.writeInt(this.width);
dataOutputStream.writeInt(level.getMinY());
dataOutputStream.writeByte(this.worldGenStep.value);
outputStream.writeInt(this.getDataDetailLevel());
outputStream.writeInt(this.width);
outputStream.writeInt(level.getMinY());
outputStream.writeByte(this.worldGenStep.value);
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
int dataDetail = dataInputStream.readInt();
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataDetail = inputStream.readInt();
if (dataDetail != dataFile.baseMetaData.dataLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.baseMetaData.dataLevel));
}
int width = dataInputStream.readInt();
int width = inputStream.readInt();
if (width != WIDTH)
{
throw new IOException(LodUtil.formatLog("Section width mismatch: "+width+" != "+ WIDTH +" (Currently only 1 section width is supported)"));
}
int minY = dataInputStream.readInt();
int minY = inputStream.readInt();
if (minY != level.getMinY())
{
LOGGER.warn("Data minY mismatch: "+minY+" != "+level.getMinY()+". Will ignore data's y level");
}
byte worldGenByte = dataInputStream.readByte();
byte worldGenByte = inputStream.readByte();
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(worldGenByte);
if (worldGenStep == null)
{
@@ -126,17 +122,14 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
@Override
public boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException
public boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
if (this.isEmpty())
{
dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE);
outputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE);
return false;
}
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
@@ -145,14 +138,14 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
{
for (int z = 0; z < this.width; z++)
{
dataOutputStream.writeInt(this.get(x, z).getSingleLength());
outputStream.writeInt(this.get(x, z).getSingleLength());
}
}
// Data array content (only on non-empty columns)
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
for (int x = 0; x < this.width; x++)
{
for (int z = 0; z < this.width; z++)
@@ -163,7 +156,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
long[] dataPointArray = columnAccessor.getRaw();
for (long dataPoint : dataPointArray)
{
dataOutputStream.writeLong(dataPoint);
outputStream.writeLong(dataPoint);
}
}
}
@@ -173,12 +166,8 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
return true;
}
@Override
public long[][] readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException
public long[][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream dataInputStream) throws IOException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
// Data array length
int dataPresentFlag = dataInputStream.readInt();
if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE)
@@ -237,28 +226,22 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
@Override
public void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException
public void writeIdMappings(DhDataOutputStream outputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(bufferedOutputStream);
outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(outputStream);
}
@Override
public FullDataPointIdMap readIdMappings(long[][] dataPoints, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException
public FullDataPointIdMap readIdMappings(long[][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // Don't close, this stream is handled outside this method
int guardByte = dataInputStream.readInt();
int guardByte = inputStream.readInt();
if (guardByte != IFullDataSource.DATA_GUARD_BYTE)
{
throw new IOException("Invalid data content end guard for ID mapping");
}
return FullDataPointIdMap.deserialize(bufferedInputStream);
return FullDataPointIdMap.deserialize(inputStream);
}
@Override
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java
+21 -39
View File
@@ -13,6 +13,7 @@ import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.objects.dataStreams.*;
import com.seibel.lod.coreapi.util.BitShiftUtil;
import com.seibel.lod.core.util.FullDataPointUtil;
import com.seibel.lod.core.util.LodUtil;
@@ -110,11 +111,8 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
@Override
public void writeSourceSummaryInfo(IDhLevel level, BufferedOutputStream bufferedOutputStream) throws IOException
public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream dataOutputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
dataOutputStream.writeShort(this.getDataDetailLevel());
dataOutputStream.writeShort(SPARSE_UNIT_DETAIL);
dataOutputStream.writeInt(SECTION_SIZE);
@@ -123,22 +121,19 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
LodUtil.assertTrue(dataFile.pos.sectionDetailLevel > SPARSE_UNIT_DETAIL);
LodUtil.assertTrue(dataFile.pos.sectionDetailLevel <= MAX_SECTION_DETAIL);
int dataDetail = dataInputStream.readShort();
int dataDetail = inputStream.readShort();
if(dataDetail != dataFile.baseMetaData.dataLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.baseMetaData.dataLevel));
}
// confirm that the detail level is correct
int sparseDetail = dataInputStream.readShort();
int sparseDetail = inputStream.readShort();
if (sparseDetail != SPARSE_UNIT_DETAIL)
{
throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}",
@@ -146,20 +141,20 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
}
// confirm the scale of the data points is correct
int sectionSize = dataInputStream.readInt();
int sectionSize = inputStream.readInt();
if (sectionSize != SECTION_SIZE)
{
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", sectionSize, SECTION_SIZE));
}
int minY = dataInputStream.readInt();
int minY = inputStream.readInt();
if (minY != level.getMinY())
{
LOGGER.warn("Data minY mismatch: "+minY+" != "+level.getMinY()+". Will ignore data's y level");
}
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(dataInputStream.readByte());
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte());
if (worldGenStep == null)
{
worldGenStep = EDhApiWorldGenerationStep.SURFACE;
@@ -173,11 +168,8 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
@Override
public boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException
public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
if (this.isEmpty)
{
dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE);
@@ -238,12 +230,8 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
return true;
}
@Override
public long[][][] readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException
public long[][][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
// calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize
// TODO these values should be constant, should we still be calculating them like this?
int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.sectionDetailLevel - SPARSE_UNIT_DETAIL);
@@ -251,7 +239,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
// check if this file has any data
int dataPresentFlag = dataInputStream.readInt();
int dataPresentFlag = inputStream.readInt();
if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE)
{
// this file is empty
@@ -265,7 +253,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
// get the number of columns (IE the bitSet from before)
int numberOfDataColumns = dataInputStream.readInt();
int numberOfDataColumns = inputStream.readInt();
// validate the number of data columns
int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent?
if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns)
@@ -276,7 +264,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
// read in the presence of each data column
byte[] bytes = new byte[numberOfDataColumns];
dataInputStream.readFully(bytes, 0, numberOfDataColumns);
inputStream.readFully(bytes, 0, numberOfDataColumns);
BitSet dataArrayIndexHasData = BitSet.valueOf(bytes);
@@ -286,7 +274,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
//====================//
// (only on non-empty columns)
int dataArrayStartByte = dataInputStream.readInt();
int dataArrayStartByte = inputStream.readInt();
// confirm the column data is starting
if (dataArrayStartByte != IFullDataSource.DATA_GUARD_BYTE)
{
@@ -309,7 +297,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
for (int x = 0; x < dataColumn.length; x++)
{
// this should be zero if the column doesn't have any data
int dataColumnLength = dataInputStream.readInt();
int dataColumnLength = inputStream.readInt();
dataColumn[x] = new long[dataColumnLength];
}
@@ -321,7 +309,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
// read in the data columns
for (int z = 0; z < dataColumn[x].length; z++)
{
dataColumn[x][z] = dataInputStream.readLong();
dataColumn[x][z] = inputStream.readLong();
}
}
}
@@ -361,23 +349,17 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
@Override
public void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException
public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(bufferedOutputStream);
this.mapping.serialize(dataOutputStream);
}
@Override
public FullDataPointIdMap readIdMappings(long[][][] dataPoints, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException
public FullDataPointIdMap readIdMappings(long[][][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // Don't close, this stream is handled outside this method
// mark the start of the ID data
int idMappingStartByte = dataInputStream.readInt();
int idMappingStartByte = inputStream.readInt();
if (idMappingStartByte != DATA_GUARD_BYTE)
{
// the file format is incorrect
@@ -385,7 +367,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
}
// deserialize the ID data
return FullDataPointIdMap.deserialize(bufferedInputStream);
return FullDataPointIdMap.deserialize(inputStream);
}
@Override
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java
+25 -43
View File
@@ -13,6 +13,7 @@ import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.objects.dataStreams.*;
import com.seibel.lod.coreapi.util.BitShiftUtil;
import com.seibel.lod.core.util.FullDataPointUtil;
import com.seibel.lod.core.util.LodUtil;
@@ -88,42 +89,36 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
@Override
public void writeSourceSummaryInfo(IDhLevel level, BufferedOutputStream bufferedOutputStream) throws IOException
public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
dataOutputStream.writeInt(this.getDataDetailLevel());
dataOutputStream.writeInt(this.width);
dataOutputStream.writeInt(level.getMinY());
dataOutputStream.writeByte(this.worldGenStep.value);
outputStream.writeInt(this.getDataDetailLevel());
outputStream.writeInt(this.width);
outputStream.writeInt(level.getMinY());
outputStream.writeByte(this.worldGenStep.value);
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
int dataDetail = dataInputStream.readInt();
int dataDetail = inputStream.readInt();
if(dataDetail != dataFile.baseMetaData.dataLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.baseMetaData.dataLevel));
}
int width = dataInputStream.readInt();
int width = inputStream.readInt();
if (width != WIDTH)
{
throw new IOException(LodUtil.formatLog("Section size mismatch: "+width+" != "+ WIDTH +" (Currently only 1 section size is supported)"));
}
int minY = dataInputStream.readInt();
int minY = inputStream.readInt();
if (minY != level.getMinY())
{
LOGGER.warn("Data minY mismatch: "+minY+" != "+level.getMinY()+". Will ignore data's y level");
}
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(dataInputStream.readByte());
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte());
if (worldGenStep == null)
{
worldGenStep = EDhApiWorldGenerationStep.SURFACE;
@@ -140,11 +135,8 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
@Override
public boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException
public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
if (this.isEmpty)
{
dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE);
@@ -174,14 +166,10 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
return true;
}
@Override
public StreamDataPointContainer readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException
public StreamDataPointContainer readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
// is source empty flag
int dataPresentFlag = dataInputStream.readInt();
int dataPresentFlag = inputStream.readInt();
if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE)
{
// Section is empty
@@ -194,7 +182,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
// data column presence
int length = dataInputStream.readInt();
int length = inputStream.readInt();
if (length < 0 || length > (WIDTH * WIDTH /8+64)*2) // TODO replace magic numbers or comment what they mean
{
throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})",
@@ -202,14 +190,14 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
}
byte[] bytes = new byte[length];
dataInputStream.readFully(bytes, 0, length);
inputStream.readFully(bytes, 0, length);
BitSet isColumnNotEmpty = BitSet.valueOf(bytes);
// Data array content
long[][] dataPointArray = new long[WIDTH * WIDTH][];
dataPresentFlag = dataInputStream.readInt();
dataPresentFlag = inputStream.readInt();
if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE)
{
throw new IOException("invalid spotty flag end guard");
@@ -217,10 +205,10 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
for (int xz = isColumnNotEmpty.nextSetBit(0); xz >= 0; xz = isColumnNotEmpty.nextSetBit(xz + 1))
{
long[] array = new long[dataInputStream.readByte()];
long[] array = new long[inputStream.readByte()];
for (int y = 0; y < array.length; y++)
{
array[y] = dataInputStream.readLong();
array[y] = inputStream.readLong();
}
dataPointArray[xz] = array;
}
@@ -248,28 +236,22 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
@Override
public void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException
public void writeIdMappings(DhDataOutputStream outputStream) throws IOException
{
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // Don't close, this stream is handled outside this method
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(bufferedOutputStream);
outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(outputStream);
}
@Override
public FullDataPointIdMap readIdMappings(StreamDataPointContainer streamDataPointContainer, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException
public FullDataPointIdMap readIdMappings(StreamDataPointContainer streamDataPointContainer, DhDataInputStream inputStream) throws IOException, InterruptedException
{
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // Don't close, this stream is handled outside this method
// Id mapping
int dataPresentFlag = dataInputStream.readInt();
int dataPresentFlag = inputStream.readInt();
if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE)
{
throw new IOException("invalid ID mapping end guard");
}
return FullDataPointIdMap.deserialize(bufferedInputStream);
return FullDataPointIdMap.deserialize(inputStream);
}
@Override
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java
+7 -6
View File
@@ -9,9 +9,9 @@ import com.seibel.lod.core.dataObjects.render.ColumnRenderSource;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.util.ArrayList;
@@ -73,16 +73,17 @@ public interface IFullDataSource
// basic stream handling //
//=======================//
// TODO make this blow up in IStreamableFullDataSource instead of the children
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
* @see IStreamableFullDataSource#writeToStream(BufferedOutputStream, IDhLevel)
* @see IStreamableFullDataSource#writeToStream(DhDataOutputStream, IDhLevel)
*/
void writeToStream(BufferedOutputStream bufferedOutputStream, IDhLevel level) throws IOException;
void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException;
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
* @see IStreamableFullDataSource#populateFromStream(FullDataMetaFile, BufferedInputStream, IDhLevel)
* @see IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel)
*/
void populateFromStream(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException;
void populateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java
+16 -18
View File
@@ -5,9 +5,8 @@ import com.seibel.lod.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.lod.core.dataObjects.fullData.accessor.FullDataArrayAccessor;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.util.objects.dataStreams.*;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
/**
@@ -19,7 +18,7 @@ import java.io.IOException;
*
* @apiNote James would've preferred to have this as an abstract class,
* however that is impossible. See the apiNote in
* {@link IStreamableFullDataSource#populateFromStream(FullDataMetaFile, BufferedInputStream, IDhLevel) populateFromStream}
* {@link IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel) populateFromStream}
* for the full reasoning.
*
* @param <SummaryDataType> defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}.
@@ -40,13 +39,13 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
* so this could've been a constructor.
* However, several inheritors of this interface already extend {@link FullDataArrayAccessor}, making that impossible.
*/
default void populateFromStream(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
default void populateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
SummaryDataType summaryData = this.readSourceSummaryInfo(dataFile, bufferedInputStream, level);
SummaryDataType summaryData = this.readSourceSummaryInfo(dataFile, inputStream, level);
this.setSourceSummaryData(summaryData);
DataContainerType dataPoints = this.readDataPoints(dataFile, summaryData.dataWidth, bufferedInputStream);
DataContainerType dataPoints = this.readDataPoints(dataFile, summaryData.dataWidth, inputStream);
if (dataPoints == null)
{
return;
@@ -54,23 +53,22 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
this.setDataPoints(dataPoints);
FullDataPointIdMap mapping = this.readIdMappings(dataPoints, bufferedInputStream);
FullDataPointIdMap mapping = this.readIdMappings(dataPoints, inputStream);
this.setIdMapping(mapping);
}
default void writeToStream(BufferedOutputStream bufferedOutputStream, IDhLevel level) throws IOException
default void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException
{
this.writeSourceSummaryInfo(level, bufferedOutputStream);
this.writeSourceSummaryInfo(level, outputStream);
boolean hasData = this.writeDataPoints(bufferedOutputStream);
boolean hasData = this.writeDataPoints(outputStream);
if (!hasData)
{
return;
}
this.writeIdMappings(bufferedOutputStream);
this.writeIdMappings(outputStream);
}
@@ -78,26 +76,26 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
/**
* Includes information about the source file that doesn't need to be saved in each data point. Like the source's size and y-level.
*/
void writeSourceSummaryInfo(IDhLevel level, BufferedOutputStream bufferedOutputStream) throws IOException;
void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException;
/**
* Confirms that the given {@link FullDataMetaFile} is valid for this {@link IStreamableFullDataSource}. <br>
* This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed.
*
* @throws IOException if the {@link FullDataMetaFile} isn't valid for this object.
*/
SummaryDataType readSourceSummaryInfo(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException;
SummaryDataType readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException;
void setSourceSummaryData(SummaryDataType summaryData);
/** @return true if any data points were present and written, false if this object was empty */
boolean writeDataPoints(BufferedOutputStream bufferedOutputStream) throws IOException;
boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException;
/** @return null if no data points were present */
DataContainerType readDataPoints(FullDataMetaFile dataFile, int width, BufferedInputStream bufferedInputStream) throws IOException;
DataContainerType readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException;
void setDataPoints(DataContainerType dataPoints);
void writeIdMappings(BufferedOutputStream bufferedOutputStream) throws IOException;
FullDataPointIdMap readIdMappings(DataContainerType dataPoints, BufferedInputStream bufferedInputStream) throws IOException, InterruptedException;
void writeIdMappings(DhDataOutputStream outputStream) throws IOException;
FullDataPointIdMap readIdMappings(DataContainerType dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException;
void setIdMapping(FullDataPointIdMap mappings);
core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java
+4 -6
View File
@@ -10,10 +10,9 @@ import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.file.renderfile.RenderMetaDataFile;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@@ -36,9 +35,8 @@ public class ColumnRenderLoader
public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException
public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
DataInputStream inputDataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
int dataFileVersion = dataFile.baseMetaData.binaryDataFormatVersion;
switch (dataFileVersion)
@@ -46,7 +44,7 @@ public class ColumnRenderLoader
case 1:
//LOGGER.info("loading render source "+dataFile.pos);
ParsedColumnData parsedColumnData = readDataV1(inputDataStream, level.getMinY());
ParsedColumnData parsedColumnData = readDataV1(inputStream, level.getMinY());
if (parsedColumnData.isEmpty)
{
LOGGER.warn("Empty render file "+dataFile.pos);
@@ -85,7 +83,7 @@ public class ColumnRenderLoader
*
* @throws IOException if there was an issue reading the stream
*/
private static ParsedColumnData readDataV1(DataInputStream inputStream, int expectedYOffset) throws IOException
private static ParsedColumnData readDataV1(DhDataInputStream inputStream, int expectedYOffset) throws IOException
{
// TODO move into ColumnRenderSource
core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderSource.java
+12 -13
View File
@@ -3,6 +3,7 @@ package com.seibel.lod.core.dataObjects.render;
import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.lod.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder;
import com.seibel.lod.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.lod.coreapi.ModInfo;
import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.lod.core.dataObjects.render.columnViews.ColumnArrayView;
@@ -11,7 +12,6 @@ import com.seibel.lod.core.dataObjects.render.columnViews.IColumnDataView;
import com.seibel.lod.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer;
import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.enums.ELodDirection;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.coreapi.util.BitShiftUtil;
@@ -203,24 +203,23 @@ public class ColumnRenderSource
// data update and output //
//========================//
public void writeData(BufferedOutputStream bufferedOutputStream) throws IOException
public void writeData(DhDataOutputStream outputStream) throws IOException
{
bufferedOutputStream.flush();
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream);
outputStream.flush();
dataOutputStream.writeByte(this.getDataDetail());
dataOutputStream.writeInt(this.verticalDataCount);
outputStream.writeByte(this.getDataDetail());
outputStream.writeInt(this.verticalDataCount);
if (this.isEmpty)
{
// no data is present
dataOutputStream.writeByte(NO_DATA_FLAG_BYTE);
outputStream.writeByte(NO_DATA_FLAG_BYTE);
}
else
{
// data is present
dataOutputStream.writeByte(DATA_GUARD_BYTE);
dataOutputStream.writeInt(this.yOffset);
outputStream.writeByte(DATA_GUARD_BYTE);
outputStream.writeInt(this.yOffset);
// write the data for each column
for (int xz = 0; xz < SECTION_SIZE * SECTION_SIZE; xz++)
@@ -228,15 +227,15 @@ public class ColumnRenderSource
for (int y = 0; y < this.verticalDataCount; y++)
{
long currentDatapoint = this.renderDataContainer[xz * this.verticalDataCount + y];
dataOutputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly
outputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly
}
}
}
dataOutputStream.writeByte(DATA_GUARD_BYTE);
dataOutputStream.writeByte(this.worldGenStep.value);
outputStream.writeByte(DATA_GUARD_BYTE);
outputStream.writeByte(this.worldGenStep.value);
bufferedOutputStream.flush();
outputStream.flush();
}
/** Overrides any data that has not been written directly using write(). Skips empty source dataPoints. */
core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java
+7 -7
View File
@@ -20,6 +20,7 @@ import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.AtomicsUtil;
import com.seibel.lod.core.util.LodUtil;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
/**
@@ -201,16 +202,16 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile
{
// Load the file.
IFullDataSource fullDataSource;
try (FileInputStream fileInputStream = this._getFileInputStream();
BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream))
try (FileInputStream fileInputStream = this.getFileInputStream();
DhDataInputStream compressedStream = new DhDataInputStream(fileInputStream))
{
fullDataSource = this.fullDataSourceLoader.loadData(this, bufferedInputStream, this.level);
fullDataSource = this.fullDataSourceLoader.loadData(this, compressedStream, this.level);
}
catch (Exception ex)
{
if (ex instanceof InterruptedException)
{
// LOGGER.warn(FullDataMetaFile.class.getSimpleName()+" loadOrGetCachedAsync interrupted.");
//LOGGER.warn(FullDataMetaFile.class.getSimpleName()+" loadOrGetCachedAsync interrupted.");
return null;
}
@@ -218,7 +219,6 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile
throw new CompletionException(ex);
}
// confirm that this thread is in control
LodUtil.assertTrue(this.inCacheWriteAccessFuture.get() == null,
"No one should be writing to the cache while we are in the process of " +
@@ -263,7 +263,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile
return future;
}
/** @return a stream for the data contained in this file, skips the metadata from {@link AbstractMetaDataContainerFile}. */
private FileInputStream _getFileInputStream() throws IOException
private FileInputStream getFileInputStream() throws IOException
{
FileInputStream fileInputStream = new FileInputStream(this.file);
@@ -456,7 +456,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile
this.baseMetaData.dataTypeId = (this.fullDataSourceLoader == null) ? 0 : this.fullDataSourceLoader.datatypeId;
this.baseMetaData.binaryDataFormatVersion = fullDataSource.getBinaryDataFormatVersion();
super.writeData((outputStream) -> fullDataSource.writeToStream(outputStream, this.level));
super.writeData((bufferedOutputStream) -> fullDataSource.writeToStream((bufferedOutputStream), this.level));
this.doesFileExist = true;
}
catch (ClosedByInterruptException e) // thrown by buffers that are interrupted
core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java
+5 -5
View File
@@ -15,9 +15,9 @@ import java.util.zip.CheckedOutputStream;
import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.lod.core.util.FileUtil;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.objects.DhUnclosableOutputStream;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream;
import org.apache.logging.log4j.Logger;
/**
@@ -188,7 +188,7 @@ public abstract class AbstractMetaDataContainerFile
}
}
protected void writeData(IMetaDataWriterFunc<BufferedOutputStream> dataWriterFunc) throws IOException
protected void writeData(IMetaDataWriterFunc<DhDataOutputStream> dataWriterFunc) throws IOException
{
LodUtil.assertTrue(this.baseMetaData != null);
if (this.file.exists())
@@ -212,10 +212,10 @@ public abstract class AbstractMetaDataContainerFile
fileChannel.position(METADATA_SIZE_IN_BYTES);
int checksum;
try(DhUnclosableOutputStream bufferedOut = new DhUnclosableOutputStream(Channels.newOutputStream(fileChannel)); // Prevent closing the channel by anything but closing the file channel
CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) // TODO: Is Adler32 ok?
try(DhDataOutputStream compressedOut = new DhDataOutputStream(Channels.newOutputStream(fileChannel));
CheckedOutputStream checkedOut = new CheckedOutputStream(compressedOut, new Adler32())) // TODO: Is Adler32 ok?
{
dataWriterFunc.writeBufferToFile(bufferedOut); // TODO it might be nice to have a DH stream we pass in instead of the base BufferedOutputStream to make it clear the streams can't be closed
dataWriterFunc.writeBufferToFile(compressedOut);
checksum = (int) checkedOut.getChecksum().getValue();
}
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java
+5 -5
View File
@@ -11,9 +11,9 @@ import com.seibel.lod.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -177,10 +177,10 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
// Load the file.
ColumnRenderSource renderSource;
try (FileInputStream fileInputStream = this.getDataContent();
BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream))
try (FileInputStream fileInputStream = this.getFileInputStream();
DhDataInputStream compressedStream = new DhDataInputStream(fileInputStream))
{
renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, bufferedInputStream, level);
renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, compressedStream, level);
}
catch (IOException ex)
{
@@ -217,7 +217,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
renderSource.getDataDetail(), renderSource.worldGenStep, RenderSourceFileHandler.RENDER_SOURCE_TYPE_ID, renderSource.getRenderDataFormatVersion());
}
private FileInputStream getDataContent() throws IOException
private FileInputStream getFileInputStream() throws IOException
{
FileInputStream fin = new FileInputStream(this.file);
int toSkip = METADATA_SIZE_IN_BYTES;
core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableInputStream.java
-14
View File
@@ -1,14 +0,0 @@
package com.seibel.lod.core.util.objects;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
public class DhUnclosableInputStream extends BufferedInputStream
{
public DhUnclosableInputStream(InputStream it) { super(it); }
@Override
public void close() throws IOException { /* Do nothing. */ }
}
core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableOutputStream.java
-14
View File
@@ -1,14 +0,0 @@
package com.seibel.lod.core.util.objects;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
public class DhUnclosableOutputStream extends BufferedOutputStream
{
public DhUnclosableOutputStream(OutputStream outputStream) { super(outputStream); }
@Override
public void close() throws IOException { /* Do nothing. */ }
}
core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataInputStream.java
+30
View File
@@ -0,0 +1,30 @@
package com.seibel.lod.core.util.objects.dataStreams;
import net.jpountz.lz4.LZ4FrameInputStream;
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* Combines multiple different streams together for ease of use
* and to prevent accidentally wrapping a stream twice or passing in
* the wrong stream. <br><br>
*
* <strong>Note:</strong>
* This stream cannot be closed,
* the passed in stream must be closed instead.
* This is done to prevent closing file channels prematurely while accessing them.
*/
public class DhDataInputStream extends DataInputStream
{
public DhDataInputStream(InputStream stream) throws IOException
{
super(new LZ4FrameInputStream(new BufferedInputStream(stream)));
}
@Override
public void close() throws IOException { /* Do nothing. */ }
}
core/src/main/java/com/seibel/lod/core/util/objects/dataStreams/DhDataOutputStream.java
+21
View File
@@ -0,0 +1,21 @@
package com.seibel.lod.core.util.objects.dataStreams;
import net.jpountz.lz4.LZ4FrameOutputStream;
import java.io.*;
/**
* See {@link DhDataInputStream} for more information about these custom streams.
* @see DhDataInputStream
*/
public class DhDataOutputStream extends DataOutputStream
{
public DhDataOutputStream(OutputStream stream) throws IOException
{
super(new LZ4FrameOutputStream(new BufferedOutputStream(stream)));
}
@Override
public void close() throws IOException { /* Do nothing. */ }
}