wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace OutputStream's with BufferedOutputStream for performance

Browse Source
This commit is contained in:
James Seibel
2023-03-09 21:43:54 -06:00
parent e5116e1ec9
commit dd8ee36487
11 changed files with 173 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/FullDataPointIdMap.java
+2 -2
View File
@@ -61,9 +61,9 @@ public class FullDataPointIdMap
}
/** Serializes all contained entries into the given stream, formatted in UTF */
public void serialize(OutputStream outputStream) throws IOException
public void serialize(BufferedOutputStream bufferedOutputStream) throws IOException
{
DataOutputStream dataStream = new DataOutputStream(outputStream); // DO NOT CLOSE! It would close all related streams
DataOutputStream dataStream = new DataOutputStream(bufferedOutputStream); // DO NOT CLOSE! It would close all related streams
dataStream.writeInt(this.entries.size());
for (Entry entry : this.entries)
{
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/IFullDataSource.java
+2 -2
View File
@@ -6,8 +6,8 @@ import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
public interface IFullDataSource
{
@@ -21,7 +21,7 @@ public interface IFullDataSource
boolean isEmpty();
void saveData(IDhLevel level, FullDataMetaFile file, OutputStream dataStream) throws IOException;
void saveData(IDhLevel level, FullDataMetaFile file, BufferedOutputStream bufferedOutputStream) throws IOException;
/**
* Attempts to get the data column for the given relative x and z position.
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java
+43 -43
View File
@@ -129,52 +129,52 @@ public class FullDataSource extends FullArrayView implements IFullDataSource
public void markNotEmpty() { this.isEmpty = false; }
@Override
public void saveData(IDhLevel level, FullDataMetaFile file, OutputStream dataStream) throws IOException
public void saveData(IDhLevel level, FullDataMetaFile file, BufferedOutputStream bufferedOutputStream) throws IOException
{
DataOutputStream dos = new DataOutputStream(dataStream); // DO NOT CLOSE
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // DO NOT CLOSE
dataOutputStream.writeInt(this.getDataDetail());
dataOutputStream.writeInt(this.size);
dataOutputStream.writeInt(level.getMinY());
if (this.isEmpty)
{
dos.writeInt(this.getDataDetail());
dos.writeInt(this.size);
dos.writeInt(level.getMinY());
if (this.isEmpty)
{
dos.writeInt(0x00000001);
return;
}
dos.writeInt(0xFFFFFFFF);
// Data array length
for (int x = 0; x < this.size; x++)
{
for (int z = 0; z < this.size; z++)
{
dos.writeInt(this.get(x, z).getSingleLength());
}
}
// Data array content (only on non-empty columns)
dos.writeInt(0xFFFFFFFF);
for (int x = 0; x < this.size; x++)
{
for (int z = 0; z < this.size; z++)
{
SingleFullArrayView column = this.get(x, z);
if (!column.doesItExist())
continue;
long[] raw = column.getRaw();
for (long l : raw)
{
dos.writeLong(l);
}
}
}
// Id mapping
dos.writeInt(0xFFFFFFFF);
this.mapping.serialize(dos);
dos.writeInt(0xFFFFFFFF);
dataOutputStream.writeInt(0x00000001);
return;
}
dataOutputStream.writeInt(0xFFFFFFFF);
// Data array length
for (int x = 0; x < this.size; x++)
{
for (int z = 0; z < this.size; z++)
{
dataOutputStream.writeInt(this.get(x, z).getSingleLength());
}
}
// Data array content (only on non-empty columns)
dataOutputStream.writeInt(0xFFFFFFFF);
for (int x = 0; x < this.size; x++)
{
for (int z = 0; z < this.size; z++)
{
SingleFullArrayView column = this.get(x, z);
if (!column.doesItExist())
continue;
long[] raw = column.getRaw();
for (long l : raw)
{
dataOutputStream.writeLong(l);
}
}
}
// Id mapping
dataOutputStream.writeInt(0xFFFFFFFF);
this.mapping.serialize(bufferedOutputStream);
dataOutputStream.writeInt(0xFFFFFFFF);
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SingleChunkFullDataSource.java
+32 -32
View File
@@ -80,42 +80,42 @@ public class SingleChunkFullDataSource extends FullArrayView implements IIncompl
public void markNotEmpty() { this.isEmpty = false; }
@Override
public void saveData(IDhLevel level, FullDataMetaFile file, OutputStream dataStream) throws IOException
public void saveData(IDhLevel level, FullDataMetaFile file, BufferedOutputStream bufferedOutputStream) throws IOException
{
DataOutputStream dos = new DataOutputStream(dataStream); // DO NOT CLOSE
{
dos.writeInt(this.getDataDetail());
dos.writeInt(this.size);
dos.writeInt(level.getMinY());
if (this.isEmpty)
{
dos.writeInt(0x00000001);
return;
}
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream); // DO NOT CLOSE
dataOutputStream.writeInt(this.getDataDetail());
dataOutputStream.writeInt(this.size);
dataOutputStream.writeInt(level.getMinY());
if (this.isEmpty)
{
dataOutputStream.writeInt(0x00000001);
return;
}
// Is column not empty
dos.writeInt(0xFFFFFFFF);
byte[] bytes = this.isColumnNotEmpty.toByteArray();
dos.writeInt(bytes.length);
dos.write(bytes);
// Is column not empty
dataOutputStream.writeInt(0xFFFFFFFF);
byte[] bytes = this.isColumnNotEmpty.toByteArray();
dataOutputStream.writeInt(bytes.length);
dataOutputStream.write(bytes);
// Data array content
dos.writeInt(0xFFFFFFFF);
for (int i = this.isColumnNotEmpty.nextSetBit(0); i >= 0; i = this.isColumnNotEmpty.nextSetBit(i + 1))
{
dos.writeByte(this.dataArrays[i].length);
if (this.dataArrays[i].length == 0)
continue;
for (long l : this.dataArrays[i]) {
dos.writeLong(l);
}
}
// Data array content
dataOutputStream.writeInt(0xFFFFFFFF);
for (int i = this.isColumnNotEmpty.nextSetBit(0); i >= 0; i = this.isColumnNotEmpty.nextSetBit(i + 1))
{
dataOutputStream.writeByte(this.dataArrays[i].length);
if (this.dataArrays[i].length == 0)
continue;
for (long l : this.dataArrays[i]) {
dataOutputStream.writeLong(l);
}
}
// Id mapping
dos.writeInt(0xFFFFFFFF);
this.mapping.serialize(dos);
dos.writeInt(0xFFFFFFFF);
}
// Id mapping
dataOutputStream.writeInt(0xFFFFFFFF);
this.mapping.serialize(bufferedOutputStream);
dataOutputStream.writeInt(0xFFFFFFFF);
}
core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/SparseFullDataSource.java
+55 -55
View File
@@ -222,73 +222,73 @@ public class SparseFullDataSource implements IIncompleteFullDataSource
//===============//
@Override
public void saveData(IDhLevel level, FullDataMetaFile file, OutputStream dataStream) throws IOException
public void saveData(IDhLevel level, FullDataMetaFile file, BufferedOutputStream bufferedOutputStream) throws IOException
{
try (DataOutputStream dos = new DataOutputStream(dataStream))
{
dos.writeShort(this.getDataDetail());
dos.writeShort(SPARSE_UNIT_DETAIL);
dos.writeInt(SECTION_SIZE);
dos.writeInt(level.getMinY());
if (this.isEmpty)
{
dos.writeInt(NO_DATA_FLAG_BYTE);
return;
}
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream);
dos.writeInt(DATA_GUARD_BYTE);
// sparse array existence bitset
BitSet dataArrayIndexHasData = new BitSet(this.sparseData.length);
for (int i = 0; i < this.sparseData.length; i++)
{
dataArrayIndexHasData.set(i, this.sparseData[i] != null);
}
byte[] bytes = dataArrayIndexHasData.toByteArray();
dos.writeInt(bytes.length);
dos.write(bytes);
dataOutputStream.writeShort(this.getDataDetail());
dataOutputStream.writeShort(SPARSE_UNIT_DETAIL);
dataOutputStream.writeInt(SECTION_SIZE);
dataOutputStream.writeInt(level.getMinY());
if (this.isEmpty)
{
dataOutputStream.writeInt(NO_DATA_FLAG_BYTE);
return;
}
dataOutputStream.writeInt(DATA_GUARD_BYTE);
// sparse array existence bitset
BitSet dataArrayIndexHasData = new BitSet(this.sparseData.length);
for (int i = 0; i < this.sparseData.length; i++)
{
dataArrayIndexHasData.set(i, this.sparseData[i] != null);
}
byte[] bytes = dataArrayIndexHasData.toByteArray();
dataOutputStream.writeInt(bytes.length);
dataOutputStream.write(bytes);
// Data array content (only on non-empty stuff)
dos.writeInt(DATA_GUARD_BYTE);
for (int i = dataArrayIndexHasData.nextSetBit(0);
i >= 0;
i = dataArrayIndexHasData.nextSetBit(i+1))
// Data array content (only on non-empty stuff)
dataOutputStream.writeInt(DATA_GUARD_BYTE);
for (int i = dataArrayIndexHasData.nextSetBit(0);
i >= 0;
i = dataArrayIndexHasData.nextSetBit(i+1))
{
// column data length
FullArrayView array = this.sparseData[i];
LodUtil.assertTrue(array != null);
for (int x = 0; x < array.width(); x++)
{
// column data length
FullArrayView array = this.sparseData[i];
LodUtil.assertTrue(array != null);
for (int x = 0; x < array.width(); x++)
for (int z = 0; z < array.width(); z++)
{
for (int z = 0; z < array.width(); z++)
{
dos.writeInt(array.get(x, z).getSingleLength());
}
dataOutputStream.writeInt(array.get(x, z).getSingleLength());
}
// column data
for (int x = 0; x < array.width(); x++)
}
// column data
for (int x = 0; x < array.width(); x++)
{
for (int z = 0; z < array.width(); z++)
{
for (int z = 0; z < array.width(); z++)
SingleFullArrayView column = array.get(x, z);
LodUtil.assertTrue(column.getMapping() == this.mapping); // the mappings must be exactly equal!
if (column.doesItExist())
{
SingleFullArrayView column = array.get(x, z);
LodUtil.assertTrue(column.getMapping() == this.mapping); // the mappings must be exactly equal!
if (column.doesItExist())
long[] raw = column.getRaw();
for (long l : raw)
{
long[] raw = column.getRaw();
for (long l : raw)
{
dos.writeLong(l);
}
dataOutputStream.writeLong(l);
}
}
}
}
// Id mapping
dos.writeInt(DATA_GUARD_BYTE);
this.mapping.serialize(dos);
dos.writeInt(DATA_GUARD_BYTE);
}
}
}
// Id mapping
dataOutputStream.writeInt(DATA_GUARD_BYTE);
this.mapping.serialize(bufferedOutputStream);
dataOutputStream.writeInt(DATA_GUARD_BYTE);
}
public static SparseFullDataSource loadData(FullDataMetaFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException, InterruptedException
core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderSource.java
+13 -12
View File
@@ -1,5 +1,6 @@
package com.seibel.lod.core.dataObjects.render;
import com.google.common.primitives.Longs;
import com.seibel.lod.core.ModInfo;
import com.seibel.lod.core.dataObjects.render.columnViews.ColumnArrayView;
import com.seibel.lod.core.dataObjects.render.columnViews.ColumnQuadView;
@@ -202,27 +203,25 @@ public class ColumnRenderSource
// data update and output //
//========================//
public void saveRender(IDhClientLevel level, RenderMetaDataFile file, OutputStream dataStream) throws IOException
public void writeData(BufferedOutputStream bufferedOutputStream) throws IOException
{
DataOutputStream dos = new DataOutputStream(dataStream); // DO NOT CLOSE
this.writeData(dos);
}
void writeData(DataOutputStream outputStream) throws IOException
{
outputStream.writeByte(this.getDataDetail());
outputStream.writeInt(this.verticalDataCount);
bufferedOutputStream.flush();
DataOutputStream dataOutputStream = new DataOutputStream(bufferedOutputStream);
dataOutputStream.writeByte(this.getDataDetail());
dataOutputStream.writeInt(this.verticalDataCount);
if (this.isEmpty)
{
// no data is present
outputStream.writeByte(NO_DATA_FLAG_BYTE);
dataOutputStream.writeByte(NO_DATA_FLAG_BYTE);
}
else
{
// data is present
outputStream.writeByte(DATA_GUARD_BYTE);
dataOutputStream.writeByte(DATA_GUARD_BYTE);
outputStream.writeInt(this.yOffset);
dataOutputStream.writeInt(this.yOffset);
// write the data for each column
for (int xz = 0; xz < SECTION_SIZE * SECTION_SIZE; xz++)
@@ -230,10 +229,12 @@ public class ColumnRenderSource
for (int y = 0; y < this.verticalDataCount; y++)
{
long currentDatapoint = this.renderDataContainer[xz * this.verticalDataCount + y];
outputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly
dataOutputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly
}
}
}
bufferedOutputStream.flush();
}
/** Overrides any data that has not been written directly using write(). Skips empty source dataPoints. */
core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java
+19 -19
View File
@@ -182,7 +182,7 @@ public abstract class AbstractMetaDataContainerFile
}
}
protected void writeData(IMetaDataWriterFunc<OutputStream> dataWriterFunc) throws IOException
protected void writeData(IMetaDataWriterFunc<BufferedOutputStream> dataWriterFunc) throws IOException
{
LodUtil.assertTrue(this.metaData != null);
if (this.file.exists())
@@ -205,33 +205,33 @@ public abstract class AbstractMetaDataContainerFile
{
file.position(METADATA_SIZE_IN_BYTES);
int checksum;
try (OutputStream channelOut = new DhUnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel
BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer?
try (//DhUnclosableOutputStream channelOut = new DhUnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel // TODO update DhUnclosableOutputStream to use a bufferedOutput, otherwise this slows down the file handling significantly
BufferedOutputStream bufferedOut = new BufferedOutputStream(Channels.newOutputStream(file)); // TODO: Is default buffer size ok? Do we even need to buffer?
CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32()))
{
// TODO: Is Adler32 ok?
dataWriterFunc.writeBufferToFile(checkedOut);
dataWriterFunc.writeBufferToFile(bufferedOut);
checksum = (int) checkedOut.getChecksum().getValue();
}
file.position(0);
// Write metadata
ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE_IN_BYTES);
buff.putInt(METADATA_IDENTITY_BYTES);
buff.putInt(this.pos.sectionX);
buff.putInt(Integer.MIN_VALUE); // Unused
buff.putInt(this.pos.sectionZ);
buff.putInt(checksum);
buff.put(this.pos.sectionDetailLevel);
buff.put(this.metaData.dataLevel);
buff.put(this.metaData.loaderVersion);
buff.put(Byte.MIN_VALUE); // Unused
buff.putLong(this.metaData.dataTypeId);
buff.putLong(Long.MAX_VALUE); //buff.putLong(this.metaData.dataVersion.get()); // not currently implemented
LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE);
buff.flip();
file.write(buff);
ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE_IN_BYTES);
buffer.putInt(METADATA_IDENTITY_BYTES);
buffer.putInt(this.pos.sectionX);
buffer.putInt(Integer.MIN_VALUE); // Unused
buffer.putInt(this.pos.sectionZ);
buffer.putInt(checksum);
buffer.put(this.pos.sectionDetailLevel);
buffer.put(this.metaData.dataLevel);
buffer.put(this.metaData.loaderVersion);
buffer.put(Byte.MIN_VALUE); // Unused
buffer.putLong(this.metaData.dataTypeId);
buffer.putLong(Long.MAX_VALUE); //buff.putLong(this.metaData.dataVersion.get()); // not currently implemented
LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE);
buffer.flip();
file.write(buffer);
file.close();
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java
+2 -2
View File
@@ -254,7 +254,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
}
}
public void save(ColumnRenderSource renderSource, IDhClientLevel level)
public void save(ColumnRenderSource renderSource)
{
if (renderSource.isEmpty())
{
@@ -272,7 +272,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile
//LOGGER.info("Saving updated render file v[{}] at sect {}", this.metaData.dataVersion.get(), this.pos);
try
{
super.writeData((out) -> renderSource.saveRender(level, this, out));
super.writeData((out) -> renderSource.writeData(out));
this.doesFileExist = true;
}
catch (IOException e)
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderSourceFileHandler.java
+3 -3
View File
@@ -310,8 +310,8 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
}
return null;
}
).thenRun(() -> this.cacheUpdateLockBySectionPos.remove(file.pos));
})
.thenRun(() -> this.cacheUpdateLockBySectionPos.remove(file.pos));
}
public ColumnRenderSource onRenderFileLoaded(ColumnRenderSource renderSource, RenderMetaDataFile file)
@@ -338,7 +338,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
file.metaData.dataLevel = currentRenderSource.getDataDetail();
file.metaData.dataTypeId = RENDER_SOURCE_TYPE_ID;
file.metaData.loaderVersion = currentRenderSource.getRenderVersion();
file.save(currentRenderSource, this.level);
file.save(currentRenderSource);
}
public void onReadRenderSourceFromCache(RenderMetaDataFile file, ColumnRenderSource data)
core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableInputStream.java
+1 -1
View File
@@ -4,7 +4,7 @@ import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
public class DhUnclosableInputStream extends FilterInputStream {
public class DhUnclosableInputStream extends FilterInputStream { // TODO replace with BufferedInputStream and this should be usable again, otherwise this significantly slows down the file handling
public DhUnclosableInputStream(InputStream it) {
super(it);
}
core/src/main/java/com/seibel/lod/core/util/objects/DhUnclosableOutputStream.java
+1 -1
View File
@@ -4,7 +4,7 @@ import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
public class DhUnclosableOutputStream extends FilterOutputStream {
public class DhUnclosableOutputStream extends FilterOutputStream { // TODO replace with BufferedInputStream and this should be usable again, otherwise this significantly slows down the file handling
public DhUnclosableOutputStream(OutputStream it) {
super(it);
}