wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Require Buffered Streams everywhere for performance

Browse Source
This commit is contained in:
James Seibel
2023-03-09 21:02:48 -06:00
parent ce472729d6
commit e5116e1ec9
11 changed files with 290 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/AbstractFullDataSourceLoader.java
+2 -1
View File
@@ -7,6 +7,7 @@ import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
@@ -59,7 +60,7 @@ public abstract class AbstractFullDataSourceLoader
* Can return null if any of the requirements aren't met.
* @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down
*/
public abstract IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException;
public abstract IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException;
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataLoader.java
+3 -2
View File
@@ -4,6 +4,7 @@ import com.seibel.lod.core.dataObjects.fullData.sources.FullDataSource;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -15,10 +16,10 @@ public class FullDataLoader extends AbstractFullDataSourceLoader
}
@Override
public IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
{
//TODO: Add decompressor here
return FullDataSource.loadData(dataFile, data, level);
return FullDataSource.loadData(dataFile, bufferedInputStream, level);
}
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java
+2 -2
View File
@@ -72,9 +72,9 @@ public class FullDataPointIdMap
}
/** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */
public static FullDataPointIdMap deserialize(InputStream inputStream) throws IOException, InterruptedException
public static FullDataPointIdMap deserialize(BufferedInputStream bufferedInputStream) throws IOException, InterruptedException
{
DataInputStream dataStream = new DataInputStream(inputStream); // DO NOT CLOSE! It would close all related streams
DataInputStream dataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE! It would close all related streams
int entityCount = dataStream.readInt();
FullDataPointIdMap newMap = new FullDataPointIdMap();
for (int i = 0; i < entityCount; i++)
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SingleChunkFullDataLoader.java
+3 -2
View File
@@ -4,6 +4,7 @@ import com.seibel.lod.core.dataObjects.fullData.sources.SingleChunkFullDataSourc
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -14,8 +15,8 @@ public class SingleChunkFullDataLoader extends AbstractFullDataSourceLoader
}
@Override
public IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
{
return SingleChunkFullDataSource.loadData(dataFile, data, level);
return SingleChunkFullDataSource.loadData(dataFile, bufferedInputStream, level);
}
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SparseFullDataLoader.java
+3 -2
View File
@@ -4,6 +4,7 @@ import com.seibel.lod.core.dataObjects.fullData.sources.SparseFullDataSource;
import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -15,8 +16,8 @@ public class SparseFullDataLoader extends AbstractFullDataSourceLoader
}
@Override
public IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
{
return SparseFullDataSource.loadData(dataFile, data, level);
return SparseFullDataSource.loadData(dataFile, bufferedInputStream, level);
}
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java
+72 -73
View File
@@ -178,82 +178,81 @@ public class FullDataSource extends FullArrayView implements IFullDataSource
}
public static FullDataSource loadData(FullDataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException, InterruptedException
public static FullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
{
DataInputStream dos = new DataInputStream(dataStream); // DO NOT CLOSE
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
int dataDetail = dataInputStream.readInt();
if (dataDetail != dataFile.metaData.dataLevel)
{
int dataDetail = dos.readInt();
if (dataDetail != dataFile.metaData.dataLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
}
int size = dos.readInt();
if (size != SECTION_SIZE)
{
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE));
}
int minY = dos.readInt();
if (minY != level.getMinY())
{
LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY());
}
int end = dos.readInt();
// Data array length
if (end == 0x00000001)
{
// Section is empty
return new FullDataSource(dataFile.pos);
}
// Non-empty section
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid header end guard");
}
long[][] data = new long[size * size][];
for (int x = 0; x < size; x++)
{
for (int z = 0; z < size; z++)
{
data[x * size + z] = new long[dos.readInt()];
}
}
// Data array content (only on non-empty columns)
end = dos.readInt();
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid data length end guard");
}
for (int i = 0; i < data.length; i++)
{
if (data[i].length == 0)
continue;
for (int j = 0; j < data[i].length; j++)
{
data[i][j] = dos.readLong();
}
}
// Id mapping
end = dos.readInt();
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid data content end guard");
}
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(new DhUnclosableInputStream(dos));
end = dos.readInt();
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid id mapping end guard");
}
return new FullDataSource(dataFile.pos, mapping, data);
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
}
int size = dataInputStream.readInt();
if (size != SECTION_SIZE)
{
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE));
}
int minY = dataInputStream.readInt();
if (minY != level.getMinY())
{
LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY());
}
int end = dataInputStream.readInt();
// Data array length
if (end == 0x00000001)
{
// Section is empty
return new FullDataSource(dataFile.pos);
}
// Non-empty section
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid header end guard");
}
long[][] data = new long[size * size][];
for (int x = 0; x < size; x++)
{
for (int z = 0; z < size; z++)
{
data[x * size + z] = new long[dataInputStream.readInt()];
}
}
// Data array content (only on non-empty columns)
end = dataInputStream.readInt();
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid data length end guard");
}
for (int i = 0; i < data.length; i++)
{
if (data[i].length == 0)
continue;
for (int j = 0; j < data[i].length; j++)
{
data[i][j] = dataInputStream.readLong();
}
}
// Id mapping
end = dataInputStream.readInt();
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid data content end guard");
}
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(bufferedInputStream);
end = dataInputStream.readInt();
if (end != 0xFFFFFFFF)
{
throw new IOException("invalid id mapping end guard");
}
return new FullDataSource(dataFile.pos, mapping, data);
}
public static FullDataSource createEmpty(DhSectionPos pos) { return new FullDataSource(pos); }
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SingleChunkFullDataSource.java
+61 -62
View File
@@ -119,74 +119,73 @@ public class SingleChunkFullDataSource extends FullArrayView implements IIncompl
}
public static SingleChunkFullDataSource loadData(FullDataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException, InterruptedException
public static SingleChunkFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
{
DataInputStream dos = new DataInputStream(dataStream); // DO NOT CLOSE
{
int dataDetail = dos.readInt();
if(dataDetail != dataFile.metaData.dataLevel)
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
int size = dos.readInt();
if (size != SECTION_SIZE)
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE));
int minY = dos.readInt();
if (minY != level.getMinY())
LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY());
int end = dos.readInt();
// Data array length
if (end == 0x00000001)
{
// Section is empty
return new SingleChunkFullDataSource(dataFile.pos);
}
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
int dataDetail = dataInputStream.readInt();
if(dataDetail != dataFile.metaData.dataLevel)
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
int size = dataInputStream.readInt();
if (size != SECTION_SIZE)
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE));
int minY = dataInputStream.readInt();
if (minY != level.getMinY())
LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY());
int end = dataInputStream.readInt();
// Data array length
if (end == 0x00000001)
{
// Section is empty
return new SingleChunkFullDataSource(dataFile.pos);
}
// Is column not empty
if (end != 0xFFFFFFFF)
throw new IOException("invalid header end guard");
int length = dos.readInt();
// Is column not empty
if (end != 0xFFFFFFFF)
throw new IOException("invalid header end guard");
int length = dataInputStream.readInt();
if (length < 0 || length > (SECTION_SIZE*SECTION_SIZE/8+64)*2)
if (length < 0 || length > (SECTION_SIZE*SECTION_SIZE/8+64)*2)
{
throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})",
length, 1, SECTION_SIZE * SECTION_SIZE / 8 + 63));
}
byte[] bytes = new byte[length];
dataInputStream.readFully(bytes, 0, length);
BitSet isColumnNotEmpty = BitSet.valueOf(bytes);
// Data array content
long[][] data = new long[SECTION_SIZE*SECTION_SIZE][];
end = dataInputStream.readInt();
if (end != 0xFFFFFFFF)
throw new IOException("invalid spotty flag end guard");
for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1))
{
long[] array = new long[dataInputStream.readByte()];
for (int j = 0; j < array.length; j++)
{
throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})",
length, 1, SECTION_SIZE * SECTION_SIZE / 8 + 63));
array[j] = dataInputStream.readLong();
}
byte[] bytes = new byte[length];
dos.readFully(bytes, 0, length);
BitSet isColumnNotEmpty = BitSet.valueOf(bytes);
data[i] = array;
}
// Data array content
long[][] data = new long[SECTION_SIZE*SECTION_SIZE][];
end = dos.readInt();
if (end != 0xFFFFFFFF)
throw new IOException("invalid spotty flag end guard");
for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1))
{
long[] array = new long[dos.readByte()];
for (int j = 0; j < array.length; j++)
{
array[j] = dos.readLong();
}
data[i] = array;
}
// Id mapping
end = dos.readInt();
if (end != 0xFFFFFFFF)
throw new IOException("invalid data content end guard");
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(new DhUnclosableInputStream(dos));
end = dos.readInt();
if (end != 0xFFFFFFFF)
throw new IOException("invalid id mapping end guard");
return new SingleChunkFullDataSource(dataFile.pos, mapping, isColumnNotEmpty, data);
}
// Id mapping
end = dataInputStream.readInt();
if (end != 0xFFFFFFFF)
throw new IOException("invalid data content end guard");
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(bufferedInputStream);
end = dataInputStream.readInt();
if (end != 0xFFFFFFFF)
throw new IOException("invalid id mapping end guard");
return new SingleChunkFullDataSource(dataFile.pos, mapping, isColumnNotEmpty, data);
}
private SingleChunkFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, BitSet isColumnNotEmpty, long[][] data)
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SparseFullDataSource.java
+134 -134
View File
@@ -291,166 +291,166 @@ public class SparseFullDataSource implements IIncompleteFullDataSource
}
}
public static SparseFullDataSource loadData(FullDataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException, InterruptedException
public static SparseFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
{
LodUtil.assertTrue(dataFile.pos.sectionDetailLevel > SPARSE_UNIT_DETAIL);
LodUtil.assertTrue(dataFile.pos.sectionDetailLevel <= MAX_SECTION_DETAIL);
DataInputStream inputStream = new DataInputStream(dataStream); // DO NOT CLOSE! It would close all related streams
{
// TODO what is a data detail?
int dataDetail = inputStream.readShort();
if(dataDetail != dataFile.metaData.dataLevel)
DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE! It would close all related streams
// TODO what is a data detail?
int dataDetail = dataInputStream.readShort();
if(dataDetail != dataFile.metaData.dataLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
}
// confirm that the detail level is correct
int sparseDetail = dataInputStream.readShort();
if (sparseDetail != SPARSE_UNIT_DETAIL)
{
throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}",
sparseDetail, SPARSE_UNIT_DETAIL)));
}
// confirm the scale of the data points is correct
int sectionSize = dataInputStream.readInt();
if (sectionSize != SECTION_SIZE)
{
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", sectionSize, SECTION_SIZE));
}
// calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize
// TODO these values should be constant, should we still be calculating them like this?
int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.sectionDetailLevel - sparseDetail);
int dataPointsPerChunk = sectionSize / chunks;
// get the data's starting Y-level
int minY = dataInputStream.readInt();
if (minY != level.getMinY())
{
LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY());
}
// check if this file has any data
int hasDataFlag = dataInputStream.readInt();
if (hasDataFlag == NO_DATA_FLAG_BYTE)
{
// this file is empty
return createEmpty(dataFile.pos);
}
else if (hasDataFlag != DATA_GUARD_BYTE)
{
// the file format is incorrect
throw new IOException("invalid header end guard");
}
else
{
// this file has data
// get the number of columns (IE the bitSet from before)
int numberOfDataColumns = dataInputStream.readInt();
// validate the number of data columns
int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent?
if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
throw new IOException(LodUtil.formatLog("Sparse Flag BitSet size outside reasonable range: {} (expects {} to {})",
numberOfDataColumns, 1, maxNumberOfDataColumns));
}
// confirm that the detail level is correct
int sparseDetail = inputStream.readShort();
if (sparseDetail != SPARSE_UNIT_DETAIL)
{
throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}",
sparseDetail, SPARSE_UNIT_DETAIL)));
}
// confirm the scale of the data points is correct
int sectionSize = inputStream.readInt();
if (sectionSize != SECTION_SIZE)
{
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", sectionSize, SECTION_SIZE));
}
// read in the presence of each data column
byte[] bytes = new byte[numberOfDataColumns];
dataInputStream.readFully(bytes, 0, numberOfDataColumns);
BitSet dataArrayIndexHasData = BitSet.valueOf(bytes);
// calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize
// TODO these values should be constant, should we still be calculating them like this?
int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.sectionDetailLevel - sparseDetail);
int dataPointsPerChunk = sectionSize / chunks;
//====================//
// Data array content //
//====================//
// get the data's starting Y-level
int minY = inputStream.readInt();
if (minY != level.getMinY())
{
LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY());
}
// check if this file has any data
int hasDataFlag = inputStream.readInt();
if (hasDataFlag == NO_DATA_FLAG_BYTE)
{
// this file is empty
return createEmpty(dataFile.pos);
}
else if (hasDataFlag != DATA_GUARD_BYTE)
// (only on non-empty columns)
int dataArrayStartByte = dataInputStream.readInt();
// confirm the column data is starting
if (dataArrayStartByte != DATA_GUARD_BYTE)
{
// the file format is incorrect
throw new IOException("invalid header end guard");
throw new IOException("invalid data length end guard");
}
else
// read in each column that has data written to it
long[][][] rawFullDataArrays = new long[chunks * chunks][][];
for (int fullDataIndex = dataArrayIndexHasData.nextSetBit(0);
fullDataIndex >= 0 && // TODO why does this happen?
fullDataIndex < rawFullDataArrays.length;
fullDataIndex = dataArrayIndexHasData.nextSetBit(fullDataIndex + 1))
{
// this file has data
long[][] dataColumn = new long[dataPointsPerChunk * dataPointsPerChunk][];
// get the number of columns (IE the bitSet from before)
int numberOfDataColumns = inputStream.readInt();
// validate the number of data columns
int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent?
if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns)
// get the column data lengths
rawFullDataArrays[fullDataIndex] = dataColumn;
for (int x = 0; x < dataColumn.length; x++)
{
throw new IOException(LodUtil.formatLog("Sparse Flag BitSet size outside reasonable range: {} (expects {} to {})",
numberOfDataColumns, 1, maxNumberOfDataColumns));
// this should be zero if the column doesn't have any data
int dataColumnLength = dataInputStream.readInt();
dataColumn[x] = new long[dataColumnLength];
}
// read in the presence of each data column
byte[] bytes = new byte[numberOfDataColumns];
inputStream.readFully(bytes, 0, numberOfDataColumns);
BitSet dataArrayIndexHasData = BitSet.valueOf(bytes);
//====================//
// Data array content //
//====================//
// (only on non-empty columns)
int dataArrayStartByte = inputStream.readInt();
// confirm the column data is starting
if (dataArrayStartByte != DATA_GUARD_BYTE)
// get the column data
for (int x = 0; x < dataColumn.length; x++)
{
// the file format is incorrect
throw new IOException("invalid data length end guard");
}
// read in each column that has data written to it
long[][][] rawFullDataArrays = new long[chunks * chunks][][];
for (int fullDataIndex = dataArrayIndexHasData.nextSetBit(0);
fullDataIndex >= 0 && // TODO why does this happen?
fullDataIndex < rawFullDataArrays.length;
fullDataIndex = dataArrayIndexHasData.nextSetBit(fullDataIndex + 1))
{
long[][] dataColumn = new long[dataPointsPerChunk * dataPointsPerChunk][];
// get the column data lengths
rawFullDataArrays[fullDataIndex] = dataColumn;
for (int x = 0; x < dataColumn.length; x++)
if (dataColumn[x].length != 0)
{
// this should be zero if the column doesn't have any data
int dataColumnLength = inputStream.readInt();
dataColumn[x] = new long[dataColumnLength];
}
// get the column data
for (int x = 0; x < dataColumn.length; x++)
{
if (dataColumn[x].length != 0)
// read in the data columns
for (int z = 0; z < dataColumn[x].length; z++)
{
// read in the data columns
for (int z = 0; z < dataColumn[x].length; z++)
{
dataColumn[x][z] = inputStream.readLong();
}
dataColumn[x][z] = dataInputStream.readLong();
}
}
}
//============//
// ID mapping //
//============//
// mark the start of the ID data
int idMappingStartByte = inputStream.readInt();
if (idMappingStartByte != DATA_GUARD_BYTE)
{
// the file format is incorrect
throw new IOException("invalid data content end guard");
}
// deserialize the ID data
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(inputStream);
int idMappingEndByte = inputStream.readInt();
if (idMappingEndByte != DATA_GUARD_BYTE)
{
// the file format is incorrect
throw new IOException("invalid id mapping end guard");
}
FullArrayView[] fullDataArrays = new FullArrayView[chunks * chunks];
for (int i = 0; i < rawFullDataArrays.length; i++)
{
if (rawFullDataArrays[i] != null)
{
fullDataArrays[i] = new FullArrayView(mapping, rawFullDataArrays[i], dataPointsPerChunk);
}
}
return new SparseFullDataSource(dataFile.pos, mapping, fullDataArrays);
}
}
//============//
// ID mapping //
//============//
// mark the start of the ID data
int idMappingStartByte = dataInputStream.readInt();
if (idMappingStartByte != DATA_GUARD_BYTE)
{
// the file format is incorrect
throw new IOException("invalid data content end guard");
}
// deserialize the ID data
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(bufferedInputStream);
int idMappingEndByte = dataInputStream.readInt();
if (idMappingEndByte != DATA_GUARD_BYTE)
{
// the file format is incorrect
throw new IOException("invalid id mapping end guard");
}
FullArrayView[] fullDataArrays = new FullArrayView[chunks * chunks];
for (int i = 0; i < rawFullDataArrays.length; i++)
{
if (rawFullDataArrays[i] != null)
{
fullDataArrays[i] = new FullArrayView(mapping, rawFullDataArrays[i], dataPointsPerChunk);
}
}
return new SparseFullDataSource(dataFile.pos, mapping, fullDataArrays);
}
}
core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java
+3 -2
View File
@@ -11,6 +11,7 @@ import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -35,9 +36,9 @@ public class ColumnRenderLoader
public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, InputStream inputStream, IDhLevel level) throws IOException
public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException
{
DataInputStream inputDataStream = new DataInputStream(inputStream); // DO NOT CLOSE
DataInputStream inputDataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE
int dataFileVersion = dataFile.metaData.loaderVersion;
switch (dataFileVersion)
core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java
+3 -2
View File
@@ -223,9 +223,10 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile
// Load the file.
IFullDataSource data;
try (FileInputStream inputStream = this.getDataContent())
try (FileInputStream fileInputStream = this.getDataContent();
BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream))
{
data = this.loader.loadData(this, inputStream, this.level);
data = this.loader.loadData(this, bufferedInputStream, this.level);
}
catch (Exception ex)
{
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java
+4 -2
View File
@@ -13,6 +13,7 @@ import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -192,9 +193,10 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
// Load the file.
ColumnRenderSource renderSource;
try (FileInputStream fio = this.getDataContent())
try (FileInputStream fileInputStream = this.getDataContent();
BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream))
{
renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, fio, level);
renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, bufferedInputStream, level);
}
catch (IOException e)
{