From e5116e1ec99ad3f0c690e2c63eef998d9dc1ef78 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 9 Mar 2023 21:02:48 -0600 Subject: [PATCH] Require Buffered Streams everywhere for performance --- .../AbstractFullDataSourceLoader.java | 3 +- .../dataObjects/fullData/FullDataLoader.java | 5 +- .../fullData/FullDataPointIdMap.java | 4 +- .../fullData/SingleChunkFullDataLoader.java | 5 +- .../fullData/SparseFullDataLoader.java | 5 +- .../fullData/sources/FullDataSource.java | 145 +++++----- .../sources/SingleChunkFullDataSource.java | 123 ++++---- .../sources/SparseFullDataSource.java | 268 +++++++++--------- .../render/ColumnRenderLoader.java | 5 +- .../file/fullDatafile/FullDataMetaFile.java | 5 +- .../file/renderfile/RenderMetaDataFile.java | 6 +- 11 files changed, 290 insertions(+), 284 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/AbstractFullDataSourceLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/AbstractFullDataSourceLoader.java index 92a526f2b..c95434268 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/AbstractFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/AbstractFullDataSourceLoader.java @@ -7,6 +7,7 @@ import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.util.*; @@ -59,7 +60,7 @@ public abstract class AbstractFullDataSourceLoader * Can return null if any of the requirements aren't met. * @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down */ - public abstract IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException; + public abstract IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException; diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataLoader.java index ba61dcf67..b2f68b352 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataLoader.java @@ -4,6 +4,7 @@ import com.seibel.lod.core.dataObjects.fullData.sources.FullDataSource; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -15,10 +16,10 @@ public class FullDataLoader extends AbstractFullDataSourceLoader } @Override - public IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException { //TODO: Add decompressor here - return FullDataSource.loadData(dataFile, data, level); + return FullDataSource.loadData(dataFile, bufferedInputStream, level); } } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java index fc882abfa..d15d3c025 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java @@ -72,9 +72,9 @@ public class FullDataPointIdMap } /** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */ - public static FullDataPointIdMap deserialize(InputStream inputStream) throws IOException, InterruptedException + public static FullDataPointIdMap deserialize(BufferedInputStream bufferedInputStream) throws IOException, InterruptedException { - DataInputStream dataStream = new DataInputStream(inputStream); // DO NOT CLOSE! It would close all related streams + DataInputStream dataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE! It would close all related streams int entityCount = dataStream.readInt(); FullDataPointIdMap newMap = new FullDataPointIdMap(); for (int i = 0; i < entityCount; i++) diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SingleChunkFullDataLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SingleChunkFullDataLoader.java index d4edc7f1f..3fdf55bab 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SingleChunkFullDataLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SingleChunkFullDataLoader.java @@ -4,6 +4,7 @@ import com.seibel.lod.core.dataObjects.fullData.sources.SingleChunkFullDataSourc import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -14,8 +15,8 @@ public class SingleChunkFullDataLoader extends AbstractFullDataSourceLoader } @Override - public IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException { - return SingleChunkFullDataSource.loadData(dataFile, data, level); + return SingleChunkFullDataSource.loadData(dataFile, bufferedInputStream, level); } } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SparseFullDataLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SparseFullDataLoader.java index de9de4305..e535a91d2 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SparseFullDataLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/SparseFullDataLoader.java @@ -4,6 +4,7 @@ import com.seibel.lod.core.dataObjects.fullData.sources.SparseFullDataSource; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -15,8 +16,8 @@ public class SparseFullDataLoader extends AbstractFullDataSourceLoader } @Override - public IFullDataSource loadData(FullDataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException { - return SparseFullDataSource.loadData(dataFile, data, level); + return SparseFullDataSource.loadData(dataFile, bufferedInputStream, level); } } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java index 4facc5f46..140b3cf4d 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java @@ -178,82 +178,81 @@ public class FullDataSource extends FullArrayView implements IFullDataSource } - public static FullDataSource loadData(FullDataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException, InterruptedException + public static FullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException { - DataInputStream dos = new DataInputStream(dataStream); // DO NOT CLOSE + DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE + + int dataDetail = dataInputStream.readInt(); + if (dataDetail != dataFile.metaData.dataLevel) { - int dataDetail = dos.readInt(); - if (dataDetail != dataFile.metaData.dataLevel) - { - throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); - } - - int size = dos.readInt(); - if (size != SECTION_SIZE) - { - throw new IOException(LodUtil.formatLog( - "Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE)); - } - - int minY = dos.readInt(); - if (minY != level.getMinY()) - { - LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY()); - } - int end = dos.readInt(); - - // Data array length - if (end == 0x00000001) - { - // Section is empty - return new FullDataSource(dataFile.pos); - } - // Non-empty section - if (end != 0xFFFFFFFF) - { - throw new IOException("invalid header end guard"); - } - - long[][] data = new long[size * size][]; - for (int x = 0; x < size; x++) - { - for (int z = 0; z < size; z++) - { - data[x * size + z] = new long[dos.readInt()]; - } - } - // Data array content (only on non-empty columns) - end = dos.readInt(); - if (end != 0xFFFFFFFF) - { - throw new IOException("invalid data length end guard"); - } - - for (int i = 0; i < data.length; i++) - { - if (data[i].length == 0) - continue; - for (int j = 0; j < data[i].length; j++) - { - data[i][j] = dos.readLong(); - } - } - // Id mapping - end = dos.readInt(); - if (end != 0xFFFFFFFF) - { - throw new IOException("invalid data content end guard"); - } - - FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(new DhUnclosableInputStream(dos)); - end = dos.readInt(); - if (end != 0xFFFFFFFF) - { - throw new IOException("invalid id mapping end guard"); - } - - return new FullDataSource(dataFile.pos, mapping, data); + throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); } + + int size = dataInputStream.readInt(); + if (size != SECTION_SIZE) + { + throw new IOException(LodUtil.formatLog( + "Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE)); + } + + int minY = dataInputStream.readInt(); + if (minY != level.getMinY()) + { + LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY()); + } + int end = dataInputStream.readInt(); + + // Data array length + if (end == 0x00000001) + { + // Section is empty + return new FullDataSource(dataFile.pos); + } + // Non-empty section + if (end != 0xFFFFFFFF) + { + throw new IOException("invalid header end guard"); + } + + long[][] data = new long[size * size][]; + for (int x = 0; x < size; x++) + { + for (int z = 0; z < size; z++) + { + data[x * size + z] = new long[dataInputStream.readInt()]; + } + } + // Data array content (only on non-empty columns) + end = dataInputStream.readInt(); + if (end != 0xFFFFFFFF) + { + throw new IOException("invalid data length end guard"); + } + + for (int i = 0; i < data.length; i++) + { + if (data[i].length == 0) + continue; + for (int j = 0; j < data[i].length; j++) + { + data[i][j] = dataInputStream.readLong(); + } + } + // Id mapping + end = dataInputStream.readInt(); + if (end != 0xFFFFFFFF) + { + throw new IOException("invalid data content end guard"); + } + + FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(bufferedInputStream); + end = dataInputStream.readInt(); + if (end != 0xFFFFFFFF) + { + throw new IOException("invalid id mapping end guard"); + } + + return new FullDataSource(dataFile.pos, mapping, data); } public static FullDataSource createEmpty(DhSectionPos pos) { return new FullDataSource(pos); } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SingleChunkFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SingleChunkFullDataSource.java index 245695f98..198260932 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SingleChunkFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SingleChunkFullDataSource.java @@ -119,74 +119,73 @@ public class SingleChunkFullDataSource extends FullArrayView implements IIncompl } - public static SingleChunkFullDataSource loadData(FullDataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException, InterruptedException + public static SingleChunkFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException { - DataInputStream dos = new DataInputStream(dataStream); // DO NOT CLOSE - { - int dataDetail = dos.readInt(); - if(dataDetail != dataFile.metaData.dataLevel) - throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); - - int size = dos.readInt(); - if (size != SECTION_SIZE) - throw new IOException(LodUtil.formatLog( - "Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE)); - - int minY = dos.readInt(); - if (minY != level.getMinY()) - LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY()); - - int end = dos.readInt(); - // Data array length - if (end == 0x00000001) - { - // Section is empty - return new SingleChunkFullDataSource(dataFile.pos); - } + DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE + + int dataDetail = dataInputStream.readInt(); + if(dataDetail != dataFile.metaData.dataLevel) + throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); + + int size = dataInputStream.readInt(); + if (size != SECTION_SIZE) + throw new IOException(LodUtil.formatLog( + "Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE)); + + int minY = dataInputStream.readInt(); + if (minY != level.getMinY()) + LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY()); + + int end = dataInputStream.readInt(); + // Data array length + if (end == 0x00000001) + { + // Section is empty + return new SingleChunkFullDataSource(dataFile.pos); + } - // Is column not empty - if (end != 0xFFFFFFFF) - throw new IOException("invalid header end guard"); - int length = dos.readInt(); + // Is column not empty + if (end != 0xFFFFFFFF) + throw new IOException("invalid header end guard"); + int length = dataInputStream.readInt(); - if (length < 0 || length > (SECTION_SIZE*SECTION_SIZE/8+64)*2) + if (length < 0 || length > (SECTION_SIZE*SECTION_SIZE/8+64)*2) + { + throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})", + length, 1, SECTION_SIZE * SECTION_SIZE / 8 + 63)); + } + + byte[] bytes = new byte[length]; + dataInputStream.readFully(bytes, 0, length); + BitSet isColumnNotEmpty = BitSet.valueOf(bytes); + + // Data array content + long[][] data = new long[SECTION_SIZE*SECTION_SIZE][]; + end = dataInputStream.readInt(); + if (end != 0xFFFFFFFF) + throw new IOException("invalid spotty flag end guard"); + + for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1)) + { + long[] array = new long[dataInputStream.readByte()]; + for (int j = 0; j < array.length; j++) { - throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})", - length, 1, SECTION_SIZE * SECTION_SIZE / 8 + 63)); + array[j] = dataInputStream.readLong(); } - - byte[] bytes = new byte[length]; - dos.readFully(bytes, 0, length); - BitSet isColumnNotEmpty = BitSet.valueOf(bytes); + data[i] = array; + } - // Data array content - long[][] data = new long[SECTION_SIZE*SECTION_SIZE][]; - end = dos.readInt(); - if (end != 0xFFFFFFFF) - throw new IOException("invalid spotty flag end guard"); - - for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1)) - { - long[] array = new long[dos.readByte()]; - for (int j = 0; j < array.length; j++) - { - array[j] = dos.readLong(); - } - data[i] = array; - } - - // Id mapping - end = dos.readInt(); - if (end != 0xFFFFFFFF) - throw new IOException("invalid data content end guard"); - - FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(new DhUnclosableInputStream(dos)); - end = dos.readInt(); - if (end != 0xFFFFFFFF) - throw new IOException("invalid id mapping end guard"); - - return new SingleChunkFullDataSource(dataFile.pos, mapping, isColumnNotEmpty, data); - } + // Id mapping + end = dataInputStream.readInt(); + if (end != 0xFFFFFFFF) + throw new IOException("invalid data content end guard"); + + FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(bufferedInputStream); + end = dataInputStream.readInt(); + if (end != 0xFFFFFFFF) + throw new IOException("invalid id mapping end guard"); + + return new SingleChunkFullDataSource(dataFile.pos, mapping, isColumnNotEmpty, data); } private SingleChunkFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, BitSet isColumnNotEmpty, long[][] data) diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SparseFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SparseFullDataSource.java index 8c5a002f0..126e53163 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SparseFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SparseFullDataSource.java @@ -291,166 +291,166 @@ public class SparseFullDataSource implements IIncompleteFullDataSource } } - public static SparseFullDataSource loadData(FullDataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException, InterruptedException + public static SparseFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException { LodUtil.assertTrue(dataFile.pos.sectionDetailLevel > SPARSE_UNIT_DETAIL); LodUtil.assertTrue(dataFile.pos.sectionDetailLevel <= MAX_SECTION_DETAIL); - DataInputStream inputStream = new DataInputStream(dataStream); // DO NOT CLOSE! It would close all related streams - { - // TODO what is a data detail? - int dataDetail = inputStream.readShort(); - if(dataDetail != dataFile.metaData.dataLevel) + DataInputStream dataInputStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE! It would close all related streams + + + // TODO what is a data detail? + int dataDetail = dataInputStream.readShort(); + if(dataDetail != dataFile.metaData.dataLevel) + { + throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); + } + + // confirm that the detail level is correct + int sparseDetail = dataInputStream.readShort(); + if (sparseDetail != SPARSE_UNIT_DETAIL) + { + throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}", + sparseDetail, SPARSE_UNIT_DETAIL))); + } + + // confirm the scale of the data points is correct + int sectionSize = dataInputStream.readInt(); + if (sectionSize != SECTION_SIZE) + { + throw new IOException(LodUtil.formatLog( + "Section size mismatch: {} != {} (Currently only 1 section size is supported)", sectionSize, SECTION_SIZE)); + } + + + // calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize + // TODO these values should be constant, should we still be calculating them like this? + int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.sectionDetailLevel - sparseDetail); + int dataPointsPerChunk = sectionSize / chunks; + + + // get the data's starting Y-level + int minY = dataInputStream.readInt(); + if (minY != level.getMinY()) + { + LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY()); + } + + + // check if this file has any data + int hasDataFlag = dataInputStream.readInt(); + if (hasDataFlag == NO_DATA_FLAG_BYTE) + { + // this file is empty + return createEmpty(dataFile.pos); + } + else if (hasDataFlag != DATA_GUARD_BYTE) + { + // the file format is incorrect + throw new IOException("invalid header end guard"); + } + else + { + // this file has data + + + // get the number of columns (IE the bitSet from before) + int numberOfDataColumns = dataInputStream.readInt(); + // validate the number of data columns + int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent? + if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns) { - throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); + throw new IOException(LodUtil.formatLog("Sparse Flag BitSet size outside reasonable range: {} (expects {} to {})", + numberOfDataColumns, 1, maxNumberOfDataColumns)); } - // confirm that the detail level is correct - int sparseDetail = inputStream.readShort(); - if (sparseDetail != SPARSE_UNIT_DETAIL) - { - throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}", - sparseDetail, SPARSE_UNIT_DETAIL))); - } - - // confirm the scale of the data points is correct - int sectionSize = inputStream.readInt(); - if (sectionSize != SECTION_SIZE) - { - throw new IOException(LodUtil.formatLog( - "Section size mismatch: {} != {} (Currently only 1 section size is supported)", sectionSize, SECTION_SIZE)); - } + // read in the presence of each data column + byte[] bytes = new byte[numberOfDataColumns]; + dataInputStream.readFully(bytes, 0, numberOfDataColumns); + BitSet dataArrayIndexHasData = BitSet.valueOf(bytes); - // calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize - // TODO these values should be constant, should we still be calculating them like this? - int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.sectionDetailLevel - sparseDetail); - int dataPointsPerChunk = sectionSize / chunks; + //====================// + // Data array content // + //====================// - // get the data's starting Y-level - int minY = inputStream.readInt(); - if (minY != level.getMinY()) - { - LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY()); - } - - - // check if this file has any data - int hasDataFlag = inputStream.readInt(); - if (hasDataFlag == NO_DATA_FLAG_BYTE) - { - // this file is empty - return createEmpty(dataFile.pos); - } - else if (hasDataFlag != DATA_GUARD_BYTE) + // (only on non-empty columns) + int dataArrayStartByte = dataInputStream.readInt(); + // confirm the column data is starting + if (dataArrayStartByte != DATA_GUARD_BYTE) { // the file format is incorrect - throw new IOException("invalid header end guard"); + throw new IOException("invalid data length end guard"); } - else + + + // read in each column that has data written to it + long[][][] rawFullDataArrays = new long[chunks * chunks][][]; + for (int fullDataIndex = dataArrayIndexHasData.nextSetBit(0); + fullDataIndex >= 0 && // TODO why does this happen? + fullDataIndex < rawFullDataArrays.length; + fullDataIndex = dataArrayIndexHasData.nextSetBit(fullDataIndex + 1)) { - // this file has data + long[][] dataColumn = new long[dataPointsPerChunk * dataPointsPerChunk][]; - - // get the number of columns (IE the bitSet from before) - int numberOfDataColumns = inputStream.readInt(); - // validate the number of data columns - int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent? - if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns) + // get the column data lengths + rawFullDataArrays[fullDataIndex] = dataColumn; + for (int x = 0; x < dataColumn.length; x++) { - throw new IOException(LodUtil.formatLog("Sparse Flag BitSet size outside reasonable range: {} (expects {} to {})", - numberOfDataColumns, 1, maxNumberOfDataColumns)); + // this should be zero if the column doesn't have any data + int dataColumnLength = dataInputStream.readInt(); + dataColumn[x] = new long[dataColumnLength]; } - // read in the presence of each data column - byte[] bytes = new byte[numberOfDataColumns]; - inputStream.readFully(bytes, 0, numberOfDataColumns); - BitSet dataArrayIndexHasData = BitSet.valueOf(bytes); - - - - //====================// - // Data array content // - //====================// - - // (only on non-empty columns) - int dataArrayStartByte = inputStream.readInt(); - // confirm the column data is starting - if (dataArrayStartByte != DATA_GUARD_BYTE) + // get the column data + for (int x = 0; x < dataColumn.length; x++) { - // the file format is incorrect - throw new IOException("invalid data length end guard"); - } - - - // read in each column that has data written to it - long[][][] rawFullDataArrays = new long[chunks * chunks][][]; - for (int fullDataIndex = dataArrayIndexHasData.nextSetBit(0); - fullDataIndex >= 0 && // TODO why does this happen? - fullDataIndex < rawFullDataArrays.length; - fullDataIndex = dataArrayIndexHasData.nextSetBit(fullDataIndex + 1)) - { - long[][] dataColumn = new long[dataPointsPerChunk * dataPointsPerChunk][]; - - // get the column data lengths - rawFullDataArrays[fullDataIndex] = dataColumn; - for (int x = 0; x < dataColumn.length; x++) + if (dataColumn[x].length != 0) { - // this should be zero if the column doesn't have any data - int dataColumnLength = inputStream.readInt(); - dataColumn[x] = new long[dataColumnLength]; - } - - // get the column data - for (int x = 0; x < dataColumn.length; x++) - { - if (dataColumn[x].length != 0) + // read in the data columns + for (int z = 0; z < dataColumn[x].length; z++) { - // read in the data columns - for (int z = 0; z < dataColumn[x].length; z++) - { - dataColumn[x][z] = inputStream.readLong(); - } + dataColumn[x][z] = dataInputStream.readLong(); } } } - - - - //============// - // ID mapping // - //============// - - // mark the start of the ID data - int idMappingStartByte = inputStream.readInt(); - if (idMappingStartByte != DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("invalid data content end guard"); - } - - // deserialize the ID data - FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(inputStream); - int idMappingEndByte = inputStream.readInt(); - if (idMappingEndByte != DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("invalid id mapping end guard"); - } - - FullArrayView[] fullDataArrays = new FullArrayView[chunks * chunks]; - for (int i = 0; i < rawFullDataArrays.length; i++) - { - if (rawFullDataArrays[i] != null) - { - fullDataArrays[i] = new FullArrayView(mapping, rawFullDataArrays[i], dataPointsPerChunk); - } - } - - return new SparseFullDataSource(dataFile.pos, mapping, fullDataArrays); } - } + + + + //============// + // ID mapping // + //============// + + // mark the start of the ID data + int idMappingStartByte = dataInputStream.readInt(); + if (idMappingStartByte != DATA_GUARD_BYTE) + { + // the file format is incorrect + throw new IOException("invalid data content end guard"); + } + + // deserialize the ID data + FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(bufferedInputStream); + int idMappingEndByte = dataInputStream.readInt(); + if (idMappingEndByte != DATA_GUARD_BYTE) + { + // the file format is incorrect + throw new IOException("invalid id mapping end guard"); + } + + FullArrayView[] fullDataArrays = new FullArrayView[chunks * chunks]; + for (int i = 0; i < rawFullDataArrays.length; i++) + { + if (rawFullDataArrays[i] != null) + { + fullDataArrays[i] = new FullArrayView(mapping, rawFullDataArrays[i], dataPointsPerChunk); + } + } + + return new SparseFullDataSource(dataFile.pos, mapping, fullDataArrays); + } } diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java index 85d8539b6..e46aa207d 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java @@ -11,6 +11,7 @@ import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; +import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -35,9 +36,9 @@ public class ColumnRenderLoader - public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, InputStream inputStream, IDhLevel level) throws IOException + public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException { - DataInputStream inputDataStream = new DataInputStream(inputStream); // DO NOT CLOSE + DataInputStream inputDataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE int dataFileVersion = dataFile.metaData.loaderVersion; switch (dataFileVersion) diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java index ed230a102..3ef85a6a4 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java @@ -223,9 +223,10 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile // Load the file. IFullDataSource data; - try (FileInputStream inputStream = this.getDataContent()) + try (FileInputStream fileInputStream = this.getDataContent(); + BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { - data = this.loader.loadData(this, inputStream, this.level); + data = this.loader.loadData(this, bufferedInputStream, this.level); } catch (Exception ex) { diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java index b4c94d8a1..b4597ab16 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java @@ -13,6 +13,7 @@ import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -192,9 +193,10 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile // Load the file. ColumnRenderSource renderSource; - try (FileInputStream fio = this.getDataContent()) + try (FileInputStream fileInputStream = this.getDataContent(); + BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { - renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, fio, level); + renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, bufferedInputStream, level); } catch (IOException e) {