wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Overhaul RenderFileHandler/FullFileHandler and remove metaData objects

Browse Source
This commit is contained in:
James Seibel
2024-01-07 16:06:01 -06:00
parent ec29ea8cc1
commit 19aedc14cd
32 changed files with 671 additions and 1433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java
+4 -4
View File
@@ -23,7 +23,7 @@ import com.google.common.collect.HashMultimap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import java.io.IOException;
import java.util.*;
@@ -120,7 +120,7 @@ public abstract class AbstractFullDataSourceLoader
//==============//
/** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */
public IFullDataSource loadTemporaryDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadTemporaryDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException
{
IFullDataSource dataSource = this.tryGetPooledSource();
if (dataSource != null)
@@ -140,9 +140,9 @@ public abstract class AbstractFullDataSourceLoader
*
* @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down
*/
public IFullDataSource loadDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException
{
IFullDataSource dataSource = this.createEmptyDataSource(dto.baseMetaData.pos);
IFullDataSource dataSource = this.createEmptyDataSource(dto.pos);
dataSource.populateFromStream(dto, dto.getInputStream(), level);
return dataSource;
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java
+6 -5
View File
@@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
@@ -64,6 +64,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
public String getDataTypeName() { return DATA_TYPE_NAME; }
private DhSectionPos sectionPos;
private boolean isEmpty = true;
public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY;
@@ -105,12 +106,12 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataDetail = inputStream.readInt();
if (dataDetail != dto.baseMetaData.dataDetailLevel)
if (dataDetail != dto.dataDetailLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetail + " != " + dto.baseMetaData.dataDetailLevel));
throw new IOException(LodUtil.formatLog("Data level mismatch. Expected: ["+dto.dataDetailLevel+"], found ["+dataDetail+"]."));
}
int width = inputStream.readInt();
@@ -187,7 +188,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
return true;
}
@Override
public long[][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream dataInputStream) throws IOException
public long[][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream dataInputStream) throws IOException
{
// Data array length
int dataPresentFlag = dataInputStream.readInt();
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java
+8 -8
View File
@@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
@@ -143,15 +143,15 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() > SPARSE_UNIT_DETAIL);
LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() <= MAX_SECTION_DETAIL);
LodUtil.assertTrue(dto.pos.getDetailLevel() > SPARSE_UNIT_DETAIL);
LodUtil.assertTrue(dto.pos.getDetailLevel() <= MAX_SECTION_DETAIL);
int dataDetailLevel = inputStream.readShort();
if (dataDetailLevel != dto.baseMetaData.dataDetailLevel)
if (dataDetailLevel != dto.dataDetailLevel)
{
throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.baseMetaData.dataDetailLevel+"]");
throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.dataDetailLevel+"]");
}
// confirm that the detail level is correct
@@ -256,11 +256,11 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
return true;
}
@Override
public long[][][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException
public long[][][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException
{
// calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize
// TODO these values should be constant, should we still be calculating them like this?
int chunks = BitShiftUtil.powerOfTwo(dto.baseMetaData.pos.getDetailLevel() - SPARSE_UNIT_DETAIL);
int chunks = BitShiftUtil.powerOfTwo(dto.pos.getDetailLevel() - SPARSE_UNIT_DETAIL);
int dataPointsPerChunk = SECTION_SIZE / chunks;
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java
+6 -6
View File
@@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
@@ -67,8 +67,8 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
@Override
public String getDataTypeName() { return DATA_TYPE_NAME; }
private DhSectionPos sectionPos;
private final BitSet isColumnNotEmpty;
private boolean isEmpty = true;
@@ -120,12 +120,12 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataDetailLevel = inputStream.readInt();
if (dataDetailLevel != dto.baseMetaData.dataDetailLevel)
if (dataDetailLevel != dto.dataDetailLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.baseMetaData.dataDetailLevel));
throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.dataDetailLevel));
}
int width = inputStream.readInt();
@@ -188,7 +188,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
return true;
}
@Override
public StreamDataPointContainer readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException
public StreamDataPointContainer readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException
{
// is source empty flag
int dataPresentFlag = inputStream.readInt();
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java
+8 -32
View File
@@ -19,18 +19,15 @@
package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader;
import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.IFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import org.jetbrains.annotations.Nullable;
import java.io.IOException;
@@ -45,7 +42,7 @@ import java.io.IOException;
* @see IIncompleteFullDataSource
* @see IStreamableFullDataSource
*/
public interface IFullDataSource
public interface IFullDataSource extends IDataSource<IDhLevel>
{
/**
* This is the byte put between different sections in the binary save file.
@@ -57,21 +54,7 @@ public interface IFullDataSource
DhSectionPos getSectionPos();
/**
* Returns the name of this data source. <br>
* Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse
* the binary data when read from file.
*/
String getDataTypeName();
/** Returns the detail level of the data contained by this {@link IFullDataSource}. */
byte getDataDetailLevel();
/** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */
byte getDataFormatVersion();
EDhApiWorldGenerationStep getWorldGenStep();
default void update(ChunkSizedFullDataAccessor chunkData, IDhLevel level) { this.update(chunkData); }
void update(ChunkSizedFullDataAccessor data);
boolean isEmpty();
@@ -110,22 +93,15 @@ public interface IFullDataSource
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
*
* @see IStreamableFullDataSource#writeToStream(DhDataOutputStream, IDhLevel)
* @see IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel)
*/
void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException;
void populateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
*
* @see IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel)
* @see IStreamableFullDataSource#repopulateFromStream(DataSourceDto, DhDataInputStream, IDhLevel)
*/
void populateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
*
* @see IStreamableFullDataSource#repopulateFromStream(MetaDataDto, DhDataInputStream, IDhLevel)
*/
void repopulateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
void repopulateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java
+1 -1
View File
@@ -77,7 +77,7 @@ public interface IIncompleteFullDataSource extends IFullDataSource
/**
* Attempts to convert this {@link IIncompleteFullDataSource} into a {@link CompleteFullDataSource}.
*
* @return a new {@link CompleteFullDataSource} if successful, this if the promotion failed, .
* @return a new {@link CompleteFullDataSource} if successful, returns itself if not.
*/
IFullDataSource tryPromotingToCompleteDataSource();
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java
+10 -10
View File
@@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
@@ -39,7 +39,7 @@ import java.io.IOException;
*
* @param <SummaryDataType> defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}.
* @param <DataContainerType> defines the object holding the data points, probably long[][] or long[][][].
* {@link IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) populateFromStream}
* {@link IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) populateFromStream}
* for the full reasoning.
*/
public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFullDataSource.FullDataSourceSummaryData, DataContainerType> extends IFullDataSource
@@ -56,11 +56,11 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
* @see IStreamableFullDataSource#populateFromStream
*/
@Override
default void repopulateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
default void repopulateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
// clear/overwrite the old data
this.resizeDataStructuresForRepopulation(dto.baseMetaData.pos);
this.getMapping().clear(dto.baseMetaData.pos);
this.resizeDataStructuresForRepopulation(dto.pos);
this.getMapping().clear(dto.pos);
// set the new data
this.populateFromStream(dto, inputStream, level);
@@ -71,7 +71,7 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
* This is expected to be used with an empty {@link IStreamableFullDataSource} and functions similar to a constructor.
*/
@Override
default void populateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
default void populateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
SummaryDataType summaryData = this.readSourceSummaryInfo(dto, inputStream, level);
this.setSourceSummaryData(summaryData);
@@ -114,19 +114,19 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
*/
void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException;
/**
* Confirms that the given {@link MetaDataDto} is valid for this {@link IStreamableFullDataSource}. <br>
* Confirms that the given {@link DataSourceDto} is valid for this {@link IStreamableFullDataSource}. <br>
* This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed.
*
* @throws IOException if the {@link MetaDataDto} isn't valid for this object.
* @throws IOException if the {@link DataSourceDto} isn't valid for this object.
*/
SummaryDataType readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException;
SummaryDataType readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException;
void setSourceSummaryData(SummaryDataType summaryData);
/** @return true if any data points were present and written, false if this object was empty */
boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException;
/** @return null if no data points were present */
DataContainerType readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException;
DataContainerType readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException;
void setDataPoints(DataContainerType dataPoints);
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
+24 -14
View File
@@ -22,6 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.render;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor;
import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
@@ -45,9 +46,9 @@ import java.util.concurrent.atomic.AtomicLong;
/**
* Stores the render data used to generate OpenGL buffers.
*
* @see RenderDataPointUtil
* @see RenderDataPointUtil
*/
public class ColumnRenderSource
public class ColumnRenderSource implements IDataSource<IDhClientLevel>
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@@ -56,7 +57,12 @@ public class ColumnRenderSource
public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
public static final byte DATA_FORMAT_VERSION = 1;
@Override
public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; }
public static final String DATA_NAME = "ColumnRenderSource";
@Override
public String getDataTypeName() { return DATA_NAME; }
/**
* This is the byte put between different sections in the binary save file.
@@ -81,6 +87,8 @@ public class ColumnRenderSource
public AtomicLong localVersion = new AtomicLong(0); // used to track changes to the data source, so that buffers can be updated when necessary
//==============//
// constructors //
//==============//
@@ -107,7 +115,7 @@ public class ColumnRenderSource
*
* @throws IOException if the DataInputStream's detail level isn't what was expected
*/
public ColumnRenderSource(DhSectionPos sectionPos, ColumnRenderLoader.ParsedColumnData parsedColumnData, IDhLevel level) throws IOException
public ColumnRenderSource(DhSectionPos sectionPos, ColumnRenderSourceLoader.ParsedColumnData parsedColumnData, IDhLevel level) throws IOException
{
if (sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET != parsedColumnData.detailLevel)
{
@@ -209,7 +217,9 @@ public class ColumnRenderSource
// data update and output //
//========================//
public void writeData(DhDataOutputStream outputStream) throws IOException
@Override
public void writeToStream(DhDataOutputStream outputStream, IDhClientLevel level) throws IOException { this.writeToStream(outputStream); }
public void writeToStream(DhDataOutputStream outputStream) throws IOException
{
outputStream.flush();
@@ -302,11 +312,8 @@ public class ColumnRenderSource
}
}
/**
* Doesn't write anything to file.
* @return true if any data was changed, false otherwise
*/
public boolean updateWithChunkData(ChunkSizedFullDataAccessor chunkDataView, IDhClientLevel level)
@Override
public void update(ChunkSizedFullDataAccessor chunkDataView, IDhClientLevel level)
{
final String errorMessagePrefix = "Unable to complete fastWrite for RenderSource pos: [" + this.sectionPos + "] and chunk pos: [" + chunkDataView.chunkPos + "]. Error:";
@@ -333,14 +340,14 @@ public class ColumnRenderSource
|| blockOffsetZ + LodUtil.CHUNK_WIDTH > this.getWidthInDataPoints())
{
LOGGER.warn(errorMessagePrefix+"Data offset is out of bounds.");
return false;
return;
}
if (Thread.interrupted())
{
LOGGER.warn(errorMessagePrefix+"write interrupted.");
return false;
return;
}
@@ -399,7 +406,7 @@ public class ColumnRenderSource
int chunksPerColumn = sourceStartingChangePos.getWidthAtDetail(chunkDataView.getSectionPos().getDetailLevel());
if (chunkDataView.getSectionPos().getX() % chunksPerColumn != 0 || chunkDataView.getSectionPos().getZ() % chunksPerColumn != 0)
{
return false; // not a multiple of the column size, so no change
return; // not a multiple of the column size, so no change
}
int relStartX = Math.floorMod(sourceStartingChangePos.x, this.getWidthInDataPoints());
int relStartZ = Math.floorMod(sourceStartingChangePos.z, this.getWidthInDataPoints());
@@ -418,8 +425,6 @@ public class ColumnRenderSource
{
this.localVersion.incrementAndGet();
}
return dataChanged;
}
@@ -442,9 +447,14 @@ public class ColumnRenderSource
public long getRoughRamUsageInBytes() { return (long) this.renderDataContainer.length * Long.BYTES; }
public DhSectionPos getSectionPos() { return this.sectionPos; }
@Override
public String getPrimaryKeyString() { return this.sectionPos.serialize(); }
public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); }
@Override
public EDhApiWorldGenerationStep getWorldGenStep() { return EDhApiWorldGenerationStep.EMPTY; }
/** @return how many data points wide this {@link ColumnRenderSource} is. */
public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); }
public byte getDetailOffset() { return SECTION_SIZE_OFFSET; }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderLoader.java → core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java
+10 -10
View File
@@ -20,9 +20,9 @@
package com.seibel.distanthorizons.core.dataObjects.render;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.file.renderfile.RenderDataMetaFile;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
@@ -31,26 +31,26 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Handles loading and parsing {@link RenderDataMetaFile}s to create {@link ColumnRenderSource}s. <br><br>
* Handles loading and parsing {@link DataSourceDto}s to create {@link ColumnRenderSource}s. <br><br>
*
* Please see the {@link ColumnRenderLoader#loadRenderSource} method to see what
* Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what
* file versions this class can handle.
*/
public class ColumnRenderLoader
public class ColumnRenderSourceLoader
{
public static ColumnRenderLoader INSTANCE = new ColumnRenderLoader();
public static ColumnRenderSourceLoader INSTANCE = new ColumnRenderSourceLoader();
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private ColumnRenderLoader() { }
private ColumnRenderSourceLoader() { }
public ColumnRenderSource loadRenderSource(RenderDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
public ColumnRenderSource loadRenderSource(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataFileVersion = dataFile.baseMetaData.binaryDataFormatVersion;
int dataFileVersion = dto.binaryDataFormatVersion;
switch (dataFileVersion)
{
@@ -60,10 +60,10 @@ public class ColumnRenderLoader
ParsedColumnData parsedColumnData = readDataV1(inputStream, level.getMinY());
if (parsedColumnData.isEmpty)
{
LOGGER.warn("Empty render file " + dataFile.pos);
LOGGER.warn("Empty render file " + dto.pos);
}
return new ColumnRenderSource(dataFile.pos, parsedColumnData, level);
return new ColumnRenderSource(dto.pos, parsedColumnData, level);
default:
throw new IOException("Invalid Data: The data version [" + dataFileVersion + "] is not supported");
}
core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java
+364
View File
@@ -0,0 +1,364 @@
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.*;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.channels.ClosedByInterruptException;
import java.nio.channels.ClosedChannelException;
import java.util.Enumeration;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
public abstract class AbstractDataSourceHandler<TDataSource extends IDataSource<TDhLevel>, TDhLevel extends IDhLevel> implements ISourceProvider<TDataSource, TDhLevel>
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final Timer DELAYED_SAVE_TIMER = new Timer();
/** How long a data source must remain un-modified before being written to disk. */
private static final int SAVE_DELAY_IN_MS = 4_000;
/**
* The highest numerical detail level known about.
* Used when determining which positions to update.
*/
protected final AtomicInteger topSectionDetailLevelRef;
protected final int minDetailLevel = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
protected final ConcurrentHashMap<DhSectionPos, TDataSource> unsavedDataSourceBySectionPos = new ConcurrentHashMap<>();
protected final ConcurrentHashMap<DhSectionPos, TimerTask> saveTimerTasksBySectionPos = new ConcurrentHashMap<>();
protected final ReentrantLock[] updateLockArray;
protected final TDhLevel level;
protected final File saveDir;
public final AbstractDataSourceRepo repo;
//=============//
// constructor //
//=============//
public AbstractDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, AbstractDataSourceRepo repo) { this(level, saveStructure, repo, null); }
public AbstractDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, AbstractDataSourceRepo repo, @Nullable File saveDirOverride)
{
this.level = level;
this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride;
if (!this.saveDir.exists() && !this.saveDir.mkdirs())
{
LOGGER.warn("Unable to create full data folder, file saving may fail.");
}
// the lock array's length is double the number of CPU cores so the number of collisions
// should be relatively low without having too many extra locks
this.updateLockArray = new ReentrantLock[Runtime.getRuntime().availableProcessors() * 2];
for (int i = 0; i < this.updateLockArray.length; i++)
{
this.updateLockArray[i] = new ReentrantLock();
}
this.repo = repo;
// determine the top detail level currently in the database
int maxSectionDetailLevel = this.repo.getMaxSectionDetailLevel();
this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel);
}
//==================//
// abstract methods //
//==================//
protected abstract TDataSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException;
/**
* Creates a new data source using any DTOs already present in the database.
* Can return null if there was an issue, but in general should return at least an empty data source.
*/
@Nullable
protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos);
protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos);
//==============//
// data reading //
//==============//
/**
* Returns the {@link TDataSource} for the given section position. <Br>
* The returned data source may be null if there was a problem. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
@Override
public CompletableFuture<TDataSource> getAsync(DhSectionPos pos)
{
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
return CompletableFuture.supplyAsync(() -> this.get(pos), executor);
}
/**
* Should only be used in internal file handler methods where we are already running on a file handler thread.
* Can return null if there was a problem.
* @see AbstractDataSourceHandler#getAsync(DhSectionPos)
*/
@Nullable
public TDataSource get(DhSectionPos pos)
{
// used the unsaved data source if present
if (this.unsavedDataSourceBySectionPos.containsKey(pos))
{
return this.unsavedDataSourceBySectionPos.get(pos);
}
// an unsaved data source isn't present
// check the database
// increase the top detail level if necessary
this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
TDataSource dataSource = null;
try
{
DataSourceDto dto = this.repo.getByPrimaryKey(pos.serialize());
if (dto != null)
{
// load from file
dataSource = this.createDataSourceFromDto(dto);
}
else
{
// attempt to create from any existing files
dataSource = this.createNewDataSourceFromExistingDtos(pos);
}
}
catch (InterruptedException ignore) { }
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e);
}
return dataSource;
}
//===============//
// data updating //
//===============//
@Override
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView)
{
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
this.recursivelyUpdateDataSourcesAsync(chunkSectionPos, chunkDataView);
}
/** Updates every data source from this position up to {@link AbstractDataSourceHandler#topSectionDetailLevelRef} */
protected void recursivelyUpdateDataSourcesAsync(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView)
{
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return;
}
// update up until we reach the highest available data source
if (pos.getDetailLevel() > this.topSectionDetailLevelRef.get())
{
return;
}
executor.execute(() ->
{
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos();
LodUtil.assertTrue(chunkSectionPos.overlapsExactly(pos), "Update failed, chunk [" + chunkSectionPos + "] does not overlap section [" + pos + "].");
// update this pos
this.updateDataSourceAtPos(pos, chunkDataView);
// recursively update the parent pos
DhSectionPos parentPos = pos.getParentPos();
this.recursivelyUpdateDataSourcesAsync(parentPos, chunkDataView);
});
}
protected void updateDataSourceAtPos(DhSectionPos pos, ChunkSizedFullDataAccessor chunkData)
{
// a lock is necessary to prevent two threads from writing to the same position at once,
// if that happens only the second update will apply and the LOD will end up with hole(s)
ReentrantLock updateLock = this.getUpdateLockForPos(pos);
try
{
updateLock.lock();
// get or create the data source
TDataSource dataSource = this.get(pos);
if (dataSource == null)
{
dataSource = this.makeEmptyDataSource(pos);
}
dataSource.update(chunkData, this.level);
this.queueDelayedSave(dataSource);
}
catch (Exception e)
{
LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e);
}
finally
{
updateLock.unlock();
}
}
/**
* Queues the given data source to save after {@link AbstractDataSourceHandler#SAVE_DELAY_IN_MS}
* milliseconds have passed without any additional modifications. <br> <br>
*
* This prevents repeatedly reading/writing the same data source to/from disk if said
* source is currently being updated via world gen or chunk modifications.
* This drastically reduces disk usage and improves performance.
*/
protected void queueDelayedSave(TDataSource dataSource)
{
DhSectionPos pos = dataSource.getSectionPos();
// put the data source in memory until it can be flushed to disk
this.unsavedDataSourceBySectionPos.put(pos, dataSource);
TimerTask task = new TimerTask()
{
@Override
public void run()
{
try
{
final TDataSource finalDataSource = AbstractDataSourceHandler.this.unsavedDataSourceBySectionPos.remove(pos);
// this can rarely happen due to imperfect concurrency handling,
// if the data source is null that just means it has already been saved so nothing needs to be done
if (finalDataSource != null)
{
AbstractDataSourceHandler.this.writeDataSourceToFile(finalDataSource);
}
}
catch (ClosedByInterruptException e) // thrown by buffers that are interrupted
{
// expected if the file handler is shut down, the exception can be ignored
}
catch (IOException e)
{
LOGGER.error("Failed to save updated data for section " + pos, e);
}
}
};
DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS);
// cancel the old save timer if present
// (this is equivalent to restarting the timer)
TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task);
if (oldTask != null)
{
oldTask.cancel();
}
}
protected void writeDataSourceToFile(TDataSource dataSource) throws IOException
{
LodUtil.assertTrue(dataSource != null);
try
{
// write the outputs to a stream to prep for writing to the database
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// the order of these streams is important, otherwise the checksum won't be calculated
CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32());
// normally a DhStream should be the topmost stream to prevent closing the stream accidentally,
// but since this stream will be closed immediately after writing anyway, it won't be an issue
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut);
dataSource.writeToStream(compressedOut, AbstractDataSourceHandler.this.level);
compressedOut.flush();
int checksum = (int) checkedOut.getChecksum().getValue();
byteArrayOutputStream.close();
// save the DTO
DataSourceDto newDto = new DataSourceDto(
dataSource.getSectionPos(), checksum,
dataSource.getDataDetailLevel(), dataSource.getWorldGenStep(), dataSource.getDataTypeName(),
dataSource.getDataFormatVersion(),
byteArrayOutputStream.toByteArray());
this.repo.save(newDto);
}
catch (ClosedChannelException e) // includes ClosedByInterruptException
{
// expected if the file handler is shut down, the exception can be ignored
}
}
//================//
// helper methods //
//================//
/** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */
protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; }
//=========//
// cleanup //
//=========//
@Override
public void close()
{
LOGGER.info("Closing ["+this.getClass().getSimpleName()+"] for level: ["+this.level+"], saving ["+this.saveTimerTasksBySectionPos.size()+"] positions.");
Enumeration<DhSectionPos> list = this.saveTimerTasksBySectionPos.keys();
while (list.hasMoreElements())
{
DhSectionPos pos = list.nextElement();
TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos);
if (saveTask != null)
{
saveTask.run();
saveTask.cancel();
}
}
LOGGER.info("["+this.getClass().getSimpleName()+"] saving complete, closing repo.");
this.repo.close();
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java
+54
View File
@@ -0,0 +1,54 @@
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.IBaseDTO;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import java.io.IOException;
/**
* Base for all data sources.
*
* @param <TDhLevel> what type of level this data source can be created from
*/
public interface IDataSource<TDhLevel extends IDhLevel> extends IBaseDTO
{
DhSectionPos getSectionPos();
@Override
default String getPrimaryKeyString() { return this.getSectionPos().serialize(); }
//===============//
// file handling //
//===============//
void update(ChunkSizedFullDataAccessor chunkData, TDhLevel level);
void writeToStream(DhDataOutputStream outputStream, TDhLevel level) throws IOException;
//===========//
// meta data //
//===========//
/** Returns the detail level of the data contained by this {@link IFullDataSource}. */
byte getDataDetailLevel();
EDhApiWorldGenerationStep getWorldGenStep();
/**
* Returns the name of this data source. <br>
* Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse
* the binary data when read from file.
*/
String getDataTypeName();
/** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */
byte getDataFormatVersion();
}
core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java
+15
View File
@@ -0,0 +1,15 @@
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import java.util.concurrent.CompletableFuture;
public interface ISourceProvider<TDataSource extends IDataSource<TDhLevel>, TDhLevel extends IDhLevel> extends AutoCloseable
{
CompletableFuture<TDataSource> getAsync(DhSectionPos pos);
void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData);
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java
+39 -331
View File
@@ -20,66 +20,31 @@
package com.seibel.distanthorizons.core.file.fullDatafile;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.sql.FullDataRepo;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.channels.ClosedByInterruptException;
import java.nio.channels.ClosedChannelException;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
public class FullDataFileHandler implements IFullDataSourceProvider
public class FullDataFileHandler extends AbstractDataSourceHandler<IFullDataSource, IDhLevel> implements IFullDataSourceProvider
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final Timer DELAYED_SAVE_TIMER = new Timer();
/** How long a data source must remain un-modified before being written to disk. */
private static final int SAVE_DELAY_IN_MS = 4_000;
protected final ConcurrentHashMap<DhSectionPos, IFullDataSource> unsavedDataSourceBySectionPos = new ConcurrentHashMap<>();
protected final ConcurrentHashMap<DhSectionPos, TimerTask> saveTimerTasksBySectionPos = new ConcurrentHashMap<>();
protected final ReentrantLock[] updateLockArray;
protected final IDhLevel level;
protected final File saveDir;
/**
* The highest numerical detail level known about.
* Used when determining which positions to update.
*/
protected final AtomicInteger topSectionDetailLevelRef;
protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET;
public final FullDataRepo fullDataRepo;
@Override
public FullDataRepo getRepo() { return this.fullDataRepo; }
@@ -90,25 +55,15 @@ public class FullDataFileHandler implements IFullDataSourceProvider
public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); }
public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride)
{
this.level = level;
this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride;
if (!this.saveDir.exists() && !this.saveDir.mkdirs())
{
LOGGER.warn("Unable to create full data folder, file saving may fail.");
}
// the lock array's length is double the number of CPU cores so the number of collisions
// should be relatively low without having too many extra locks
this.updateLockArray = new ReentrantLock[Runtime.getRuntime().availableProcessors() * 2];
for (int i = 0; i < this.updateLockArray.length; i++)
{
this.updateLockArray[i] = new ReentrantLock();
}
super(level, saveStructure, createRepo(level, saveStructure), saveDirOverride);
}
private static FullDataRepo createRepo(IDhLevel level, AbstractSaveStructure saveStructure)
{
File saveDir = saveStructure.getFullDataFolder(level.getLevelWrapper());
try
{
this.fullDataRepo = new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
return new FullDataRepo("jdbc:sqlite", saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
}
catch (SQLException e)
{
@@ -116,81 +71,23 @@ public class FullDataFileHandler implements IFullDataSourceProvider
// or the database update failed
throw new RuntimeException(e);
}
// determine the top detail level currently in the database
int maxSectionDetailLevel = this.fullDataRepo.getMaxSectionDetailLevel();
this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel);
}
//==============//
// data reading //
//==============//
//====================//
// Abstract overrides //
//====================//
/**
* Returns the {@link IFullDataSource} for the given section position. <Br>
* The returned data source may be null. <Br> <Br>
*
* For now, if result is null, it prob means error has occurred when loading or creating the file object. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
@Override
public CompletableFuture<IFullDataSource> getAsync(DhSectionPos pos)
protected IFullDataSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException
{
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
return CompletableFuture.supplyAsync(() -> this.get(pos), executor);
}
/**
* Should be used in internal methods where we are already running on a file handler thread.
* @see FullDataFileHandler#getAsync(DhSectionPos)
*/
protected IFullDataSource get(DhSectionPos pos)
{
// used the unsaved data source if present
if (this.unsavedDataSourceBySectionPos.containsKey(pos))
{
return this.unsavedDataSourceBySectionPos.get(pos);
}
// an unsaved data source isn't present
// check the database
// increase the top detail level if necessary
this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
IFullDataSource dataSource = null;
try
{
MetaDataDto dto = this.fullDataRepo.getByPrimaryKey(pos.serialize());
if (dto != null)
{
// load from file
AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.baseMetaData.dataType, dto.baseMetaData.binaryDataFormatVersion);
dataSource = loader.loadDataSource(dto, this.level);
}
else
{
// attempt to create from any existing files
dataSource = this.createNewDataSourceFromExistingDtos(pos);
}
}
catch (InterruptedException ignore) { }
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e);
}
AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.dataType, dto.binaryDataFormatVersion);
IFullDataSource dataSource = loader.loadDataSource(dto, this.level);
return dataSource;
}
/** Creates a new data source using any DTOs already present in the database. */
@Override
protected IFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos)
{
IIncompleteFullDataSource newFullDataSource = this.makeEmptyDataSource(pos);
@@ -205,7 +102,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
}
// TODO replace with a SQL query, it should be much faster
// get all non-empty sections to sample from
ArrayList<DhSectionPos> samplePosList = new ArrayList<>();
ArrayList<DhSectionPos> possibleChildList = new ArrayList<>();
pos.forEachChild((childPos) ->
@@ -218,7 +115,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
while (possibleChildList.size() != 0)
{
DhSectionPos possiblePos = possibleChildList.remove(possibleChildList.size()-1);
if (this.fullDataRepo.existsWithPrimaryKey(possiblePos.serialize()))
if (this.repo.existsWithPrimaryKey(possiblePos.serialize()))
{
samplePosList.add(possiblePos);
}
@@ -268,200 +165,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
return newFullDataSource.tryPromotingToCompleteDataSource();
}
//===============//
// data updating //
//===============//
@Override
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView)
{
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET);
this.recursivelyUpdateDataSourcesAsync(chunkSectionPos, chunkDataView);
}
/** Updates every data source from this position up to {@link FullDataFileHandler#topSectionDetailLevelRef} */
private void recursivelyUpdateDataSourcesAsync(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView)
{
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return;
}
// update up until we reach the highest available data source
if (pos.getDetailLevel() > this.topSectionDetailLevelRef.get())
{
return;
}
executor.execute(() ->
{
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos();
LodUtil.assertTrue(chunkSectionPos.overlapsExactly(pos), "Update failed, chunk [" + chunkSectionPos + "] does not overlap section [" + pos + "].");
// update this pos
this.updateDataSourceAtPos(pos, chunkDataView);
// recursively update the parent pos
DhSectionPos parentPos = pos.getParentPos();
this.recursivelyUpdateDataSourcesAsync(parentPos, chunkDataView);
});
}
private void updateDataSourceAtPos(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView)
{
// a lock is necessary to prevent two threads from writing to the same position at once,
// if that happens only the second update will apply and the LOD will end up with hole(s)
ReentrantLock updateLock = this.getUpdateLockForPos(pos);
try
{
updateLock.lock();
// get or create the data source
IFullDataSource fullDataSource = this.get(pos);
if (fullDataSource == null)
{
fullDataSource = this.makeEmptyDataSource(pos);
}
fullDataSource.update(chunkDataView);
// try promoting the datasource
if (fullDataSource instanceof IIncompleteFullDataSource)
{
IIncompleteFullDataSource incompleteFullDataSource = (IIncompleteFullDataSource) fullDataSource;
fullDataSource = incompleteFullDataSource.tryPromotingToCompleteDataSource();
}
this.queueDelayedSave(fullDataSource);
}
catch (Exception e)
{
LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e);
}
finally
{
updateLock.unlock();
}
}
/**
* Queues the given data source to save after {@link FullDataFileHandler#SAVE_DELAY_IN_MS}
* milliseconds have passed without any additional modifications. <br> <br>
*
* This prevents repeatedly reading/writing the same data source to/from disk if said
* source is currently being updated via world gen or chunk modifications.
* This drastically reduces disk usage and improves performance.
*/
private void queueDelayedSave(IFullDataSource fullDataSource)
{
DhSectionPos pos = fullDataSource.getSectionPos();
// put the data source in memory until it can be flushed to disk
this.unsavedDataSourceBySectionPos.put(pos, fullDataSource);
TimerTask task = new TimerTask()
{
@Override
public void run()
{
try
{
final IFullDataSource finalFullDataSource = FullDataFileHandler.this.unsavedDataSourceBySectionPos.remove(pos);
// this can rarely happen due to imperfect concurrency handling,
// if the data source is null that just means it has already been saved so nothing needs to be done
if (finalFullDataSource != null)
{
FullDataFileHandler.this.writeDataSourceToFile(finalFullDataSource, (bufferedOutputStream) ->
{
try
{
finalFullDataSource.writeToStream(bufferedOutputStream, FullDataFileHandler.this.level);
}
catch (Exception e)
{
// if this try catch isn't included an empty exception will be thrown instead, which makes debugging extremely painful
LOGGER.error("Error writing data stream for pos: [" + finalFullDataSource.getSectionPos() + "], error: " + e.getMessage(), e);
}
});
}
}
catch (ClosedByInterruptException e) // thrown by buffers that are interrupted
{
// expected if the file handler is shut down, the exception can be ignored
//LOGGER.warn("FullData file writing interrupted.", e);
}
catch (IOException e)
{
LOGGER.error("Failed to save updated data for section " + pos, e);
}
}
};
DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS);
//
TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task);
if (oldTask != null)
{
oldTask.cancel();
}
}
private void writeDataSourceToFile(IFullDataSource fullDataSource, AbstractMetaDataContainerFile.IMetaDataWriterFunc<DhDataOutputStream> dataWriterFunc) throws IOException
{
LodUtil.assertTrue(fullDataSource != null);
boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get();
if (showFullDataFileStatus)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW),
0.2, 16f));
}
try
{
// write the outputs to a stream to prep for writing to the database
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// the order of these streams is important, otherwise the checksum won't be calculated
CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32());
// normally a DhStream should be the topmost stream to prevent closing the stream accidentally,
// but since this stream will be closed immediately after writing anyway, it won't be an issue
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut);
dataWriterFunc.writeBinaryDataToStream(compressedOut);
compressedOut.flush();
int checksum = (int) checkedOut.getChecksum().getValue();
byteArrayOutputStream.close();
// save the DTO
BaseMetaData baseMetaData = new BaseMetaData(fullDataSource.getSectionPos(), checksum,
fullDataSource.getDataDetailLevel(), fullDataSource.getWorldGenStep(), fullDataSource.getDataTypeName(),
fullDataSource.getDataFormatVersion());
MetaDataDto newDto = new MetaDataDto(baseMetaData, byteArrayOutputStream.toByteArray());
this.fullDataRepo.save(newDto);
}
catch (ClosedChannelException e) // includes ClosedByInterruptException
{
// expected if the file handler is shut down, the exception can be ignored
//LOGGER.warn(AbstractMetaDataContainerFile.class.getSimpleName()+" file writing interrupted. Error: "+e.getMessage());
}
}
//================//
// helper methods //
//================//
/** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */
protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; }
protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos)
{
return pos.getDetailLevel() <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL ?
@@ -471,29 +175,33 @@ public class FullDataFileHandler implements IFullDataSourceProvider
//=========//
// cleanup //
//=========//
//===================//
// extension methods //
//===================//
@Override
public void close()
public void writeDataSourceToFile(IFullDataSource fullDataSource) throws IOException
{
LOGGER.info("Closing file handler for level: ["+this.level+"], saving ["+this.saveTimerTasksBySectionPos.size()+"] positions.");
Enumeration<DhSectionPos> list = this.saveTimerTasksBySectionPos.keys();
while (list.hasMoreElements())
// doing this here guarantees that all changes are caught and promoted
if (fullDataSource instanceof IIncompleteFullDataSource)
{
DhSectionPos pos = list.nextElement();
TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos);
if (saveTask != null)
{
saveTask.run();
saveTask.cancel();
}
fullDataSource = ((IIncompleteFullDataSource) fullDataSource).tryPromotingToCompleteDataSource();
}
LOGGER.info("File handler saving complete, closing repo.");
this.fullDataRepo.close();
super.writeDataSourceToFile(fullDataSource);
// save has completed
boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get();
if (showFullDataFileStatus)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW),
0.2, 16f));
}
}
@Override
public int getUnsavedDataSourceCount() { return this.unsavedDataSourceBySectionPos.size(); }
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java
+2 -2
View File
@@ -66,7 +66,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
//===========//
@Override
protected IFullDataSource get(DhSectionPos pos)
public IFullDataSource get(DhSectionPos pos)
{
IFullDataSource dataSource = super.get(pos);
@@ -256,7 +256,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
{
if (chunkSizedFullDataSource.getSectionPos().overlapsExactly(this.loadedTargetFullDataSource.getSectionPos()))
{
((DhLevel) level).saveWrites(chunkSizedFullDataSource);
((DhLevel) level).updateDataSourcesWithChunkData(chunkSizedFullDataSource);
//GeneratedFullDataFileHandler.this.write(this.loadedTargetFullDataSource.getSectionPos(), chunkSizedFullDataSource);
}
};
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java
+7 -2
View File
@@ -21,6 +21,10 @@ package com.seibel.distanthorizons.core.file.fullDatafile;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.ISourceProvider;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.FullDataRepo;
@@ -30,12 +34,13 @@ import java.util.concurrent.CompletableFuture;
* Handles reading, writing, and updating {@link IFullDataSource}'s. <br>
* Should be backed by a database handled by a {@link FullDataRepo}.
*/
public interface IFullDataSourceProvider extends AutoCloseable
public interface IFullDataSourceProvider extends ISourceProvider<IFullDataSource, IDhLevel>, AutoCloseable
{
CompletableFuture<IFullDataSource> getAsync(DhSectionPos pos);
IFullDataSource get(DhSectionPos pos);
void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData);
FullDataRepo getRepo();
int getUnsavedDataSourceCount();
}
core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java
-149
View File
@@ -1,149 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.metaData;
import java.io.*;
import java.nio.channels.ClosedChannelException;
import java.nio.file.*;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.AbstractDhRepo;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import org.apache.logging.log4j.Logger;
/**
* This represents the data appended to any file we write. <br>
* Contains a {@link BaseMetaData} which holds most of the necessary values written to the file. <br><br>
*
* Used size: 40 bytes <br>
* Remaining space: 24 bytes <br>
* Total size: 64 bytes <br><br><br>
*
*
* <Strong>Metadata format: </Strong><br><br>
* <code>
* 4 bytes: metadata identifier bytes: "DHv0" (in ascii: 0x44 48 76 30) this signals the file is in the metadata format <br>
* 4 bytes: section X position <br>
* 4 bytes: section Y position (Unused, for future proofing) <br>
* 4 bytes: section Z position <br> <br>
*
* 4 bytes: data checksum <br>
* 1 byte: section detail level <br>
* 1 byte: data detail level // Note: not sure if this is needed <br>
* 1 byte: loader version <br>
* 1 byte: unused <br> <br>
*
* 8 bytes: datatype identifier <br> <br>
*
* 8 bytes: data version
* </code>
*/
public abstract class AbstractMetaDataContainerFile
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
/**
* Will be null if no file exists for this object. <br>
* NOTE: Only use {@link BaseMetaData#pos} when initially setting up this object, afterwards the standalone {@link AbstractMetaDataContainerFile#pos} should be used.
*/
public volatile BaseMetaData baseMetaData = null;
/** Should be used instead of the position inside {@link AbstractMetaDataContainerFile#baseMetaData} */
public final DhSectionPos pos;
//==============//
// constructors //
//==============//
/**
* Create a metaFile in this path.
*
* @throws FileAlreadyExistsException If the path already has a file.
*/
protected AbstractMetaDataContainerFile(DhSectionPos pos) { this.pos = pos; }
/**
* Creates an {@link AbstractMetaDataContainerFile} with the file at the given path.
*
* @throws IOException if the file was formatted incorrectly
* @throws FileNotFoundException if no file exists for the given path
*/
protected AbstractMetaDataContainerFile(BaseMetaData baseMetaData) throws IOException
{
this.baseMetaData = baseMetaData;
this.pos = this.baseMetaData.pos;
}
//==============//
// file writing //
//==============//
public void writeToDatabase(IMetaDataWriterFunc<DhDataOutputStream> dataWriterFunc, AbstractDhRepo<MetaDataDto> repo) throws IOException
{
LodUtil.assertTrue(this.baseMetaData != null);
try
{
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// the order of these streams is important, otherwise the checksum won't be calculated
CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32());
// normally a DhStream should be the topmost stream to prevent closing the stream accidentally, but since this stream will be closed immediately after writing anyway, it won't be an issue
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut);
// write the contained data
dataWriterFunc.writeBinaryDataToStream(compressedOut);
compressedOut.flush();
this.baseMetaData.checksum = (int) checkedOut.getChecksum().getValue();
byteArrayOutputStream.close();
MetaDataDto dto = new MetaDataDto(this.baseMetaData, byteArrayOutputStream.toByteArray());
repo.save(dto);
}
catch (ClosedChannelException e) // includes ClosedByInterruptException
{
// expected if the file handler is shut down, the exception can be ignored
//LOGGER.warn(AbstractMetaDataContainerFile.class.getSimpleName()+" file writing interrupted. Error: "+e.getMessage());
}
}
//================//
// helper classes //
//================//
/** TODO replace with a method that accepts a {@link DhDataOutputStream} and writes to that instead */
@FunctionalInterface
public interface IMetaDataWriterFunc<T> { void writeBinaryDataToStream(T t) throws IOException; }
}
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java
+9 -11
View File
@@ -20,29 +20,27 @@
package com.seibel.distanthorizons.core.file.renderfile;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.file.ISourceProvider;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.sql.FullDataRepo;
import com.seibel.distanthorizons.core.sql.RenderDataRepo;
import java.util.concurrent.CompletableFuture;
/**
* This represents LOD data that is stored in long term storage (IE LOD files stored on the hard drive) <br>
* Example: {@link RenderSourceFileHandler RenderSourceFileHandler} <br><br>
*
* This is used to create {@link ColumnRenderSource}'s
* Handles reading, writing, and updating {@link ColumnRenderSource}'s. <br>
* Should be backed by a database handled by a {@link RenderDataRepo}.
*/
public interface IRenderSourceProvider extends AutoCloseable
public interface IRenderSourceProvider extends ISourceProvider<ColumnRenderSource, IDhClientLevel>
{
CompletableFuture<ColumnRenderSource> readAsync(DhSectionPos pos);
CompletableFuture<ColumnRenderSource> getAsync(DhSectionPos pos);
void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData);
CompletableFuture<Void> flushAndSaveAsync();
void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData);
/** Deletes any data stored in the render cache so it can be re-created */
void deleteRenderCache();
RenderDataRepo getRepo();
}
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java
-493
View File
@@ -1,493 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.renderfile;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderLoader;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.AtomicsUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.*;
import java.lang.ref.SoftReference;
import java.util.Random;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
/** Represents a File that contains a {@link ColumnRenderSource}. */
public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final boolean ALWAYS_INVALIDATE_CACHE = false;
public static final String RENDER_SOURCE_TYPE = ColumnRenderSource.DATA_NAME;
/**
* Can be cleared if the garbage collector determines there isn't enough space. <br><br>
*
* When clearing, don't set to null, instead create a SoftReference containing null.
* This makes null checks simpler.
*/
private SoftReference<ColumnRenderSource> cachedRenderDataSourceRef = new SoftReference<>(null);
private final AtomicReference<CompletableFuture<ColumnRenderSource>> renderSourceLoadFutureRef = new AtomicReference<>(null);
private final IDhClientLevel clientLevel;
private final IFullDataSourceProvider fullDataSourceProvider;
private final IRenderSourceProvider renderDataSourceProvider;
private boolean doesDtoExist;
//=============//
// constructor //
//=============//
/**
* NOTE: should only be used if there is NOT an existing file.
* @throws IOException if a file already exists for this position
*/
public static RenderDataMetaFile createNewFileForPos(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, DhSectionPos pos) throws IOException { return new RenderDataMetaFile(fullDataSourceProvider, renderDataSourceProvider, clientLevel, pos); }
private RenderDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, DhSectionPos pos) throws IOException
{
super(pos);
this.fullDataSourceProvider = fullDataSourceProvider;
this.renderDataSourceProvider = renderDataSourceProvider;
this.clientLevel = clientLevel;
LodUtil.assertTrue(this.baseMetaData == null);
this.doesDtoExist = false;
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus);
}
/**
* NOTE: should only be used if there IS an existing file.
* @throws IOException if no file exists for this position
*/
public static RenderDataMetaFile createFromExistingFile(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, MetaDataDto metaDataDto) throws IOException { return new RenderDataMetaFile(fullDataSourceProvider, renderDataSourceProvider, clientLevel, metaDataDto); }
private RenderDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, MetaDataDto metaDataDto) throws IOException
{
super(metaDataDto.baseMetaData);
this.fullDataSourceProvider = fullDataSourceProvider;
this.renderDataSourceProvider = renderDataSourceProvider;
this.clientLevel = clientLevel;
LodUtil.assertTrue(this.baseMetaData != null);
this.doesDtoExist = true;
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus);
}
//=============//
// data update //
//=============//
public void updateChunkIfSourceExistsAsync(ChunkSizedFullDataAccessor chunkDataView)
{
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos();
LodUtil.assertTrue(this.pos.overlapsExactly(chunkSectionPos), "Chunk pos " + chunkSectionPos + " doesn't overlap with section " + this.pos);
// update the render source if one exists
CompletableFuture<ColumnRenderSource> renderSourceLoadFuture = this.getCachedDataSourceAsync(true);
if (renderSourceLoadFuture == null)
{
return;
}
renderSourceLoadFuture.thenAccept((renderSource) ->
{
boolean dataUpdated = renderSource.updateWithChunkData(chunkDataView, this.clientLevel);
// add a debug particle
boolean showRenderDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus.get();
if (showRenderDataFileStatus)
{
float offset = new Random(System.nanoTime() ^ Thread.currentThread().getId()).nextFloat() * 16f;
Color debugColor = dataUpdated ? Color.blue : Color.red;
DebugRenderer.makeParticle(
new DebugRenderer.BoxParticle(
new DebugRenderer.Box(chunkDataView.getSectionPos(), 32f, 64f + offset, 0.07f, debugColor),
2.0, 16f
)
);
}
});
}
//======================//
// render source getter //
//======================//
public CompletableFuture<ColumnRenderSource> getOrLoadCachedDataSourceAsync(Executor fileReaderThreads)
{
CompletableFuture<ColumnRenderSource> renderSourceLoadFuture = this.getCachedDataSourceAsync(true);
if (renderSourceLoadFuture != null)
{
// return the in-process future
return renderSourceLoadFuture;
}
else
{
// there is no cached data, we'll have to load it
renderSourceLoadFuture = new CompletableFuture<>();
if (!this.renderSourceLoadFutureRef.compareAndSet(null, renderSourceLoadFuture))
{
// two threads attempted to start this job at the same time, only use the first future
renderSourceLoadFuture = this.renderSourceLoadFutureRef.get();
}
}
final CompletableFuture<ColumnRenderSource> getSourceFuture = renderSourceLoadFuture;
if (!this.doesDtoExist)
{
// create a new Meta file and render source
// create an empty render source
byte dataDetailLevel = (byte) (this.pos.getDetailLevel() - ColumnRenderSource.SECTION_SIZE_OFFSET);
int verticalSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(dataDetailLevel);
ColumnRenderSource newColumnRenderSource = new ColumnRenderSource(this.pos, verticalSize, this.clientLevel.getMinY());
this.baseMetaData = new BaseMetaData(
newColumnRenderSource.getSectionPos(), -1, newColumnRenderSource.getDataDetailLevel(),
newColumnRenderSource.worldGenStep, RENDER_SOURCE_TYPE,
newColumnRenderSource.getRenderDataFormatVersion());
this.updateRenderCacheAsync(newColumnRenderSource).whenComplete((voidObj, ex) ->
{
this.cachedRenderDataSourceRef = new SoftReference<>(newColumnRenderSource);
this.renderSourceLoadFutureRef.set(null);
getSourceFuture.complete(newColumnRenderSource);
});
}
else
{
// load the existing Meta file and render source
CompletableFuture.supplyAsync(() ->
{
if (this.baseMetaData == null)
{
throw new IllegalStateException("Meta data not loaded!");
}
// Load the render source file.
ColumnRenderSource renderSource;
try (InputStream inputStream = this.getInputStream(); // throws IoException
DhDataInputStream compressedInputStream = new DhDataInputStream(inputStream))
{
renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, compressedInputStream, this.clientLevel);
}
catch (IOException ex)
{
throw new CompletionException(ex);
}
return renderSource;
}, fileReaderThreads)
// TODO: Check for file version and only update if needed.
.thenCompose(renderSource -> this.updateRenderCacheAsync(renderSource))
.whenComplete((renderSource, ex) ->
{
if (ex != null)
{
if (!LodUtil.isInterruptOrReject(ex))
{
LOGGER.error("Error loading pos: "+this.pos+": ", ex);
}
// set the render source to null to prevent instances where a corrupt or incomplete render source is returned
renderSource = null;
}
this.renderSourceLoadFutureRef.set(null);
this.cachedRenderDataSourceRef = new SoftReference<>(renderSource);
getSourceFuture.complete(renderSource);
});
}
return getSourceFuture;
}
// TODO merge with FullDataMetaFile
private InputStream getInputStream() throws IOException
{
MetaDataDto dto = this.renderDataSourceProvider.getRepo().getByPrimaryKey(this.pos.serialize());
return new ByteArrayInputStream(dto.dataArray);
}
//===============//
// cache handler //
//===============//
public CompletableFuture<ColumnRenderSource> updateRenderCacheAsync(ColumnRenderSource renderSource)
{
DebugRenderer.BoxWithLife debugBox = new DebugRenderer.BoxWithLife(new DebugRenderer.Box(renderSource.sectionPos, 74f, 86f, 0.1f, Color.red), 1.0, 32f, Color.green.darker());
//// Skip updating the cache if the data file is already up-to-date
//FullDataMetaFile dataFile = this.fullDataSourceProvider.getDtoIfExist(this.pos);
//if (!ALWAYS_INVALIDATE_CACHE && dataFile != null && dataFile.baseMetaData != null && dataFile.baseMetaData.checksum == this.baseMetaData.dataVersion.get()) // TODO can we make it so the version comparisons either both use the checksum or the dataVersion? Comparing checksum and dataVersion is kinda confusing
//{
// LOGGER.debug("Skipping render cache update for " + this.pos);
// renderSource.localVersion.incrementAndGet();
// return CompletableFuture.completedFuture(renderSource);
//}
// get the full data source
CompletableFuture<IFullDataSource> fullDataSourceFuture =
this.fullDataSourceProvider.getAsync(renderSource.getSectionPos())
.thenApply((fullDataSource) ->
{
debugBox.box.color = Color.yellow.darker();
//// get the metaFile's version
//FullDataMetaFile renderSourceMetaFile = this.fullDataSourceProvider.getDtoIfExist(this.pos);
//if (renderSourceMetaFile != null && renderSourceMetaFile.baseMetaData != null)
//{
// renderDataVersionRef.value = renderSourceMetaFile.baseMetaData.checksum;
//}
return fullDataSource;
}).exceptionally((ex) ->
{
LOGGER.error("Exception when getting data for updateCache()", ex);
return null;
});
// convert the full data source into a render source
CompletableFuture<ColumnRenderSource> transformFuture = fullDataSourceFuture
.handle((fullDataSource, ex) ->
{
if (ex == null)
{
ColumnRenderSource newRenderSource = null;
try
{
newRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.clientLevel);
}
catch (Exception e)
{
LOGGER.error("Unable to transform full data to render data for pos: "+this.pos, e);
}
try
{
if (newRenderSource != null)
{
renderSource.updateFromRenderSource(newRenderSource);
// update the meta data
this.baseMetaData.dataVersion.set(Integer.MAX_VALUE);
this.baseMetaData.dataDetailLevel = renderSource.getDataDetailLevel();
this.baseMetaData.dataType = RENDER_SOURCE_TYPE;
this.baseMetaData.binaryDataFormatVersion = renderSource.getRenderDataFormatVersion();
// save to file
this.save(renderSource);
}
}
catch (Throwable e)
{
LOGGER.error("Exception when writing render data for pos: "+this.pos, e);
}
}
else if (!LodUtil.isInterruptOrReject(ex))
{
LOGGER.error("Exception when updating render file using data source: ", ex);
}
debugBox.close();
return renderSource;
});
return transformFuture;
}
//===============//
// file handling //
//===============//
public CompletableFuture<Void> flushAndSaveAsync()
{
if (!this.renderDataSourceProvider.getRepo().existsWithPrimaryKey(this.pos.serialize()))
{
return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist.
}
// FIXME: TODO: Change updateRenderSource to true. Currently is false cause a dead future making render handler hang,
// and that render cache aren't actually used really yet due to missing versioning atm. So disabling for now.
CompletableFuture<ColumnRenderSource> getSourceFuture = this.getCachedDataSourceAsync(false);
if (getSourceFuture == null)
{
return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save.
}
// Wait for the data to be read, which also flushes changes to the file.
return getSourceFuture.thenAccept((columnRenderSource) -> { /* discard the render source, it doesn't need to be returned */ });
}
/** writes the given {@link ColumnRenderSource} to file */
private void save(ColumnRenderSource renderSource)
{
if (renderSource.isEmpty())
{
// delete the empty data source
this.fullDataSourceProvider.getRepo().deleteByPrimaryKey(this.pos.serialize());
this.doesDtoExist = false;
}
else
{
//LOGGER.info("Saving updated render file v[{}] at sect {}", this.metaData.dataVersion.get(), this.pos);
try
{
super.writeToDatabase((dhDataOutputStream) -> renderSource.writeData(dhDataOutputStream), this.renderDataSourceProvider.getRepo());
this.doesDtoExist = true;
}
catch (IOException e)
{
LOGGER.error("Failed to save updated render data for pos "+this.pos, e);
}
}
}
//=======//
// debug //
//=======//
@Override
public void debugRender(DebugRenderer debugRenderer)
{
if (this.cachedRenderDataSourceRef.get() != null)
{
return;
//color = Color.GREEN;
}
// determine the color
Color color = Color.black;
if (this.renderSourceLoadFutureRef.get() != null)
{
color = Color.BLUE;
}
else if (this.doesDtoExist)
{
color = Color.RED;
}
debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 64, 72, 0.05f, color));
}
//================//
// helper methods //
//================//
/** @return returns null if {@link RenderDataMetaFile#renderSourceLoadFutureRef} is empty and no cached {@link ColumnRenderSource} exists. */
@Nullable
private CompletableFuture<ColumnRenderSource> getCachedDataSourceAsync(boolean updateRenderSourceCache)
{
// check if another thread is already loading the data source
CompletableFuture<ColumnRenderSource> renderSourceLoadFuture = this.renderSourceLoadFutureRef.get();
if (renderSourceLoadFuture != null)
{
return renderSourceLoadFuture;
}
// attempt to get the cached render source
ColumnRenderSource cachedRenderDataSource = this.cachedRenderDataSourceRef.get();
if (cachedRenderDataSource == null)
{
// no cached data exists and no one is trying to load it
return null;
}
else
{
// cached data exists
if (!updateRenderSourceCache)
{
// just return the render source
return CompletableFuture.completedFuture(cachedRenderDataSource);
}
// update the render cache, wait for the update to finish, then return the render source
// Create a new future if one doesn't already exist
CompletableFuture<ColumnRenderSource> newFuture = new CompletableFuture<>();
CompletableFuture<ColumnRenderSource> oldFuture = AtomicsUtil.compareAndExchange(this.renderSourceLoadFutureRef, null, newFuture);
if (oldFuture != null)
{
// An update is already in progress, return its future.
return oldFuture;
}
else
{
this.updateRenderCacheAsync(cachedRenderDataSource)
// wait for the handler to finish before returning the renderSource
.handle((ignoredRenderSource, ex) ->
{
newFuture.complete(cachedRenderDataSource);
this.renderSourceLoadFutureRef.set(null);
return null;
});
return newFuture;
}
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java
+45 -241
View File
@@ -20,7 +20,10 @@
package com.seibel.distanthorizons.core.file.renderfile;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandler;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
@@ -28,9 +31,8 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.sql.DataSourceDto;
import com.seibel.distanthorizons.core.sql.RenderDataRepo;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import org.apache.logging.log4j.Logger;
@@ -39,28 +41,15 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
public class RenderSourceFileHandler implements IRenderSourceProvider
public class RenderSourceFileHandler extends AbstractDataSourceHandler<ColumnRenderSource, IDhClientLevel> implements IRenderSourceProvider
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private final F3Screen.NestedMessage threadPoolMsg;
protected final ConcurrentHashMap<DhSectionPos, RenderDataMetaFile> loadedMetaFileBySectionPos = new ConcurrentHashMap<>();
private final IDhClientLevel clientLevel;
private final File saveDir;
/** This is the lowest (highest numeric) detail level that this {@link RenderSourceFileHandler} is keeping track of. */
AtomicInteger topDetailLevelRef = new AtomicInteger(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
private final IFullDataSourceProvider fullDataSourceProvider;
private final WeakHashMap<CompletableFuture<?>, ETaskType> taskTracker = new WeakHashMap<>();
public final RenderDataRepo renderDataRepo;
@Override
public RenderDataRepo getRepo() { return this.renderDataRepo; }
//=============//
@@ -69,20 +58,18 @@ public class RenderSourceFileHandler implements IRenderSourceProvider
public RenderSourceFileHandler(IFullDataSourceProvider sourceProvider, IDhClientLevel clientLevel, AbstractSaveStructure saveStructure)
{
super(clientLevel, saveStructure, createRepo(clientLevel, saveStructure));
this.fullDataSourceProvider = sourceProvider;
this.clientLevel = clientLevel;
this.saveDir = saveStructure.getRenderCacheFolder(clientLevel.getLevelWrapper());
if (!this.saveDir.exists() && !this.saveDir.mkdirs())
{
LOGGER.warn("Unable to create render data folder, file saving may fail.");
}
this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log);
}
private static RenderDataRepo createRepo(IDhClientLevel clientLevel, AbstractSaveStructure saveStructure)
{
File saveDir = saveStructure.getRenderCacheFolder(clientLevel.getLevelWrapper());
try
{
this.renderDataRepo = new RenderDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
return new RenderDataRepo("jdbc:sqlite", saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
}
catch (SQLException e)
{
@@ -94,169 +81,36 @@ public class RenderSourceFileHandler implements IRenderSourceProvider
//===============//
// file handling //
//===============//
//====================//
// Abstract overrides //
//====================//
/** This call is thread safe and can be called concurrently from multiple threads. */
@Override
public CompletableFuture<ColumnRenderSource> readAsync(DhSectionPos pos)
@Override
protected ColumnRenderSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException
{ return ColumnRenderSourceLoader.INSTANCE.loadRenderSource(dto, dto.getInputStream(), this.level); }
@Override
protected ColumnRenderSource createNewDataSourceFromExistingDtos(DhSectionPos pos)
{
// don't continue if the handler has been shut down
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor != null && executor.isTerminated())
ColumnRenderSource renderDataSource;
IFullDataSource fullDataSource = this.fullDataSourceProvider.get(pos);
if (fullDataSource != null)
{
return CompletableFuture.completedFuture(null);
renderDataSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level);
}
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
RenderDataMetaFile metaFile = this.getLoadOrMakeFile(pos);
if (metaFile == null)
else
{
return CompletableFuture.completedFuture(ColumnRenderSource.createEmptyRenderSource(pos));
renderDataSource = this.makeEmptyDataSource(pos);
}
CompletableFuture<ColumnRenderSource> getDataSourceFuture = metaFile.getOrLoadCachedDataSourceAsync(executor)
.handle((renderSource, exception) ->
{
if (exception != null)
{
LOGGER.error("Uncaught error in readAsync for pos: " + pos + ". Error:", exception);
}
return (renderSource != null) ? renderSource : ColumnRenderSource.createEmptyRenderSource(pos);
});
synchronized (this.taskTracker)
{
this.taskTracker.put(getDataSourceFuture, ETaskType.READ);
}
return getDataSourceFuture;
}
/** @return null if there was an issue */
private RenderDataMetaFile getLoadOrMakeFile(DhSectionPos pos)
{
RenderDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos);
if (metaFile != null)
{
return metaFile;
}
MetaDataDto metaDataDto = this.renderDataRepo.getByPrimaryKey(pos.serialize());
if (metaDataDto != null)
{
synchronized (this)
{
// A file exists, but isn't loaded yet.
// Double check locking for loading file, as loading file means also loading the metadata, which
// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
metaFile = this.loadedMetaFileBySectionPos.get(pos);
if (metaFile != null)
{
return metaFile; // someone else loaded it already.
}
try
{
metaFile = RenderDataMetaFile.createFromExistingFile(this.fullDataSourceProvider, this, this.clientLevel, metaDataDto);
this.topDetailLevelRef.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.getDetailLevel()));
this.loadedMetaFileBySectionPos.put(pos, metaFile);
return metaFile;
}
catch (IOException e)
{
LOGGER.error("Failed to read meta data file at pos " + pos + ": ", e);
this.renderDataRepo.delete(metaDataDto);
}
}
}
// File does not exist, create it.
// In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS
// to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects.
try
{
metaFile = RenderDataMetaFile.createNewFileForPos(this.fullDataSourceProvider, this, this.clientLevel, pos);
}
catch (IOException e)
{
LOGGER.error("IOException on creating new render data file at "+pos, e);
return null;
}
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
// This is a Compare And Swap with expected null value.
RenderDataMetaFile metaFileCas = this.loadedMetaFileBySectionPos.putIfAbsent(pos, metaFile);
return (metaFileCas == null) ? metaFile : metaFileCas;
return renderDataSource;
}
@Override
protected ColumnRenderSource makeEmptyDataSource(DhSectionPos pos)
{ return ColumnRenderSource.createEmptyRenderSource(pos); }
//=============//
// data saving //
//=============//
/**
* This call is thread safe and can be called concurrently from multiple threads. <br>
* This allows fast writes of new data to the render source, without having to wait for the data to be written to disk.
*/
@Override
public void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkDataView)
{
// convert to the lowest detail level so all detail levels are updated
this.writeChunkDataToFileRecursively(chunkDataView, DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
this.fullDataSourceProvider.updateDataSourcesWithChunkData(chunkDataView);
}
private void writeChunkDataToFileRecursively(ChunkSizedFullDataAccessor chunk, byte sectionDetailLevel)
{
DhSectionPos boundingPos = chunk.getSectionPos();
DhSectionPos minSectionPos = boundingPos.convertNewToDetailLevel(sectionDetailLevel);
DhSectionPos.DhMutableSectionPos fileSectionPos = new DhSectionPos.DhMutableSectionPos((byte)0, 0, 0);
int width = (sectionDetailLevel > boundingPos.getDetailLevel()) ? 1 : boundingPos.getWidthCountForLowerDetailedSection(sectionDetailLevel);
for (int xOffset = 0; xOffset < width; xOffset++)
{
for (int zOffset = 0; zOffset < width; zOffset++)
{
fileSectionPos.mutate(sectionDetailLevel, minSectionPos.getX() + xOffset, minSectionPos.getZ() + zOffset);
RenderDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(fileSectionPos); // bypass the getLoadOrMakeFile() since we only want cached files.
if (metaFile != null)
{
metaFile.updateChunkIfSourceExistsAsync(chunk);
}
}
}
if (sectionDetailLevel < this.topDetailLevelRef.get())
{
this.writeChunkDataToFileRecursively(chunk, (byte) (sectionDetailLevel + 1));
}
}
/** This call is thread safe and can be called concurrently from multiple threads. */
@Override
public CompletableFuture<Void> flushAndSaveAsync()
{
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
for (RenderDataMetaFile metaFile : this.loadedMetaFileBySectionPos.values())
{
futures.add(metaFile.flushAndSaveAsync());
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
//=========//
// F3 menu //
@@ -266,85 +120,35 @@ public class RenderSourceFileHandler implements IRenderSourceProvider
private String[] f3Log()
{
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
String queueSize = executor != null ? executor.getQueue().size()+"" : "-";
String completedTaskSize = executor != null ? executor.getCompletedTaskCount()+"" : "-";
String queueSize = (executor != null) ? executor.getQueue().size()+"" : "-";
String completedTaskSize = (executor != null) ? executor.getCompletedTaskCount()+"" : "-";
ArrayList<String> lines = new ArrayList<>();
lines.add("Render Source File Handler [" + this.clientLevel.getClientLevelWrapper().getDimensionType().getDimensionName() + "]");
lines.add(" Loaded files: " + this.loadedMetaFileBySectionPos.size());
lines.add("File Handler [" + this.level.getLevelWrapper().getDimensionType().getDimensionName() + "]");
lines.add(" Thread pool tasks: " + queueSize + " (completed: " + completedTaskSize + ")");
lines.add(" Unsaved render sources: " + this.unsavedDataSourceBySectionPos.size());
lines.add(" Unsaved data sources: " + this.fullDataSourceProvider.getUnsavedDataSourceCount());
int totalFutures = this.taskTracker.size();
EnumMap<ETaskType, Integer> tasksOutstanding = new EnumMap<>(ETaskType.class);
EnumMap<ETaskType, Integer> tasksCompleted = new EnumMap<>(ETaskType.class);
for (ETaskType type : ETaskType.values())
{
tasksOutstanding.put(type, 0);
tasksCompleted.put(type, 0);
}
synchronized (this.taskTracker)
{
for (Map.Entry<CompletableFuture<?>, ETaskType> entry : this.taskTracker.entrySet())
{
if (entry.getKey().isDone())
{
tasksCompleted.put(entry.getValue(), tasksCompleted.get(entry.getValue()) + 1);
}
else
{
tasksOutstanding.put(entry.getValue(), tasksOutstanding.get(entry.getValue()) + 1);
}
}
}
int totalOutstanding = tasksOutstanding.values().stream().mapToInt(Integer::intValue).sum();
lines.add(" Futures: " + totalFutures + " (outstanding: " + totalOutstanding + ")");
for (ETaskType type : ETaskType.values())
{
lines.add(" " + type + ": " + tasksOutstanding.get(type) + " / " + (tasksOutstanding.get(type) + tasksCompleted.get(type)));
}
return lines.toArray(new String[0]);
}
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView)
{
super.updateDataSourcesWithChunkData(chunkDataView);
this.fullDataSourceProvider.updateDataSourcesWithChunkData(chunkDataView);
}
//=====================//
// clearing / shutdown //
// shutdown / clearing //
//=====================//
@Override
public void close()
{
LOGGER.info("Closing " + this.getClass().getSimpleName() + " with [" + this.loadedMetaFileBySectionPos.size() + "] files...");
super.close();
this.threadPoolMsg.close();
this.renderDataRepo.close();
}
public void deleteRenderCache()
{
// clear the cached files
this.loadedMetaFileBySectionPos.clear();
// delete the render cache
this.renderDataRepo.deleteAll();
}
//================//
// helper classes //
//================//
/**
* READ <br>
* UPDATE_READ_DATA <br>
* UPDATE <br>
* ON_LOADED <br>
*/
private enum ETaskType
{
READ, UPDATE_READ_DATA, UPDATE, ON_LOADED,
}
public void deleteRenderCache() { this.repo.deleteAll(); }
}
core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java
+3 -6
View File
@@ -30,7 +30,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.distanthorizons.core.render.LodQuadTree;
import com.seibel.distanthorizons.core.render.RenderBufferHandler;
import com.seibel.distanthorizons.core.render.renderer.LodRenderer;
@@ -42,7 +41,6 @@ import com.seibel.distanthorizons.coreapi.util.math.Mat4f;
import org.apache.logging.log4j.Logger;
import java.io.Closeable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicReference;
public class ClientLevelModule implements Closeable
@@ -178,14 +176,13 @@ public class ClientLevelModule implements Closeable
//===============//
// data handling //
//===============//
public void writeChunkDataToFile(ChunkSizedFullDataAccessor data)
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data)
{
DhSectionPos pos = data.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET);
ClientRenderState ClientRenderState = this.ClientRenderStateRef.get();
if (ClientRenderState != null)
{
ClientRenderState.renderSourceFileHandler.writeChunkDataToFile(pos, data);
ClientRenderState.renderSourceFileHandler.updateDataSourcesWithChunkData(data);
ClientRenderState.quadtree.reloadPos(data.sectionPos);
}
else
{
core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java
+1 -2
View File
@@ -35,7 +35,6 @@ import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.util.concurrent.CompletableFuture;
/** The level used when connected to a server */
public class DhClientLevel extends DhLevel implements IDhClientLevel
@@ -116,7 +115,7 @@ public class DhClientLevel extends DhLevel implements IDhClientLevel
public ILevelWrapper getLevelWrapper() { return levelWrapper; }
@Override
public void saveWrites(ChunkSizedFullDataAccessor data) { this.clientside.writeChunkDataToFile(data); }
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); }
@Override
public int getMinY() { return levelWrapper.getMinHeight(); }
core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java
+1 -1
View File
@@ -186,7 +186,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS
public boolean hasSkyLight() { return this.serverLevelWrapper.hasSkyLight(); }
@Override
public void saveWrites(ChunkSizedFullDataAccessor data) { this.clientside.writeChunkDataToFile(data); }
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); }
@Override
public int getMinY() { return getLevelWrapper().getMinHeight(); }
core/src/main/java/com/seibel/distanthorizons/core/level/DhLevel.java
+2 -2
View File
@@ -34,7 +34,7 @@ public abstract class DhLevel implements IDhLevel
protected DhLevel() { this.chunkToLodBuilder = new ChunkToLodBuilder(); }
public abstract void saveWrites(ChunkSizedFullDataAccessor data);
public abstract void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data);
@Override
@@ -54,7 +54,7 @@ public abstract class DhLevel implements IDhLevel
return;
}
this.saveWrites(chunkSizedFullDataAccessor);
this.updateDataSourcesWithChunkData(chunkSizedFullDataAccessor);
ApiEventInjector.INSTANCE.fireAllEvents(
DhApiChunkModifiedEvent.class,
new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunk.getChunkPos().x, chunk.getChunkPos().z));
core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java
+1 -1
View File
@@ -50,7 +50,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
public void serverTick() { this.chunkToLodBuilder.tick(); }
@Override
public void saveWrites(ChunkSizedFullDataAccessor data)
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data)
{
DhSectionPos pos = data.getSectionPos();
pos = pos.convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET);
core/src/main/java/com/seibel/distanthorizons/core/level/IDhServerLevel.java
-1
View File
@@ -19,7 +19,6 @@
package com.seibel.distanthorizons.core.level;
import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper;
public interface IDhServerLevel extends IDhWorldGenLevel
core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java
-3
View File
@@ -417,10 +417,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
}
}
// delete the cache files
// TODO this will only delete the files for this level/world
this.renderSourceProvider.deleteRenderCache();
LOGGER.info("Render cache invalidated, please wait a moment for everything to reload...");
}
catch (Exception e)
core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java
+1 -1
View File
@@ -178,7 +178,7 @@ public class LodRenderSection implements IDebugRenderable
private void startLoadRenderSourceAsync()
{
this.renderSourceLoadFuture = this.renderSourceProvider.readAsync(this.pos);
this.renderSourceLoadFuture = this.renderSourceProvider.getAsync(this.pos);
this.renderSourceLoadFuture.whenComplete((renderSource, ex) ->
{
this.renderSource = renderSource;
core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java → core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java
+24 -26
View File
@@ -20,19 +20,17 @@
package com.seibel.distanthorizons.core.sql;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.coreapi.util.StringUtil;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Map;
public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
public abstract class AbstractDataSourceRepo extends AbstractDhRepo<DataSourceDto>
{
public AbstractMetaDataRepo(String databaseType, String databaseLocation) throws SQLException
public AbstractDataSourceRepo(String databaseType, String databaseLocation) throws SQLException
{
super(databaseType, databaseLocation, MetaDataDto.class);
super(databaseType, databaseLocation, DataSourceDto.class);
}
@@ -46,7 +44,7 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
//=======================//
@Override
public MetaDataDto convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
public DataSourceDto convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
{
String posString = (String) objectMap.get("DhSectionPos");
DhSectionPos pos = DhSectionPos.deserialize(posString);
@@ -61,22 +59,22 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
String dataType = (String) objectMap.get("DataType");
byte binaryDataFormatVersion = (Byte) objectMap.get("BinaryDataFormatVersion");
BaseMetaData baseMetaData = new BaseMetaData(pos,
checksum, dataDetailLevel, worldGenStep,
dataType, binaryDataFormatVersion);
// binary data
byte[] dataByteArray = (byte[]) objectMap.get("Data");
MetaDataDto metaFile = new MetaDataDto(baseMetaData, dataByteArray);
return metaFile;
DataSourceDto dto = new DataSourceDto(
pos,
checksum, dataDetailLevel, worldGenStep,
dataType, binaryDataFormatVersion,
dataByteArray);
return dto;
}
@Override
public String createSelectPrimaryKeySql(String primaryKey) { return "SELECT * FROM "+this.getTableName()+" WHERE DhSectionPos = '"+primaryKey+"'"; }
@Override
public PreparedStatement createInsertStatement(MetaDataDto dto) throws SQLException
public PreparedStatement createInsertStatement(DataSourceDto dto) throws SQLException
{
String sql =
"INSERT INTO "+this.getTableName() + "\n" +
@@ -94,12 +92,12 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
int i = 1;
statement.setObject(i++, dto.getPrimaryKeyString());
statement.setObject(i++, dto.baseMetaData.checksum);
statement.setObject(i++, dto.baseMetaData.dataVersion);
statement.setObject(i++, dto.baseMetaData.dataDetailLevel);
statement.setObject(i++, dto.baseMetaData.worldGenStep);
statement.setObject(i++, dto.baseMetaData.dataType);
statement.setObject(i++, dto.baseMetaData.binaryDataFormatVersion);
statement.setObject(i++, dto.checksum);
statement.setObject(i++, dto.dataVersion);
statement.setObject(i++, dto.dataDetailLevel);
statement.setObject(i++, dto.worldGenStep);
statement.setObject(i++, dto.dataType);
statement.setObject(i++, dto.binaryDataFormatVersion);
statement.setObject(i++, dto.dataArray);
@@ -107,7 +105,7 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
}
@Override
public PreparedStatement createUpdateStatement(MetaDataDto dto) throws SQLException
public PreparedStatement createUpdateStatement(DataSourceDto dto) throws SQLException
{
String sql =
"UPDATE "+this.getTableName()+" \n" +
@@ -126,12 +124,12 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
PreparedStatement statement = this.createPreparedStatement(sql);
int i = 1;
statement.setObject(i++, dto.baseMetaData.checksum);
statement.setObject(i++, dto.baseMetaData.dataVersion);
statement.setObject(i++, dto.baseMetaData.dataDetailLevel);
statement.setObject(i++, dto.baseMetaData.worldGenStep);
statement.setObject(i++, dto.baseMetaData.dataType);
statement.setObject(i++, dto.baseMetaData.binaryDataFormatVersion);
statement.setObject(i++, dto.checksum);
statement.setObject(i++, dto.dataVersion);
statement.setObject(i++, dto.dataDetailLevel);
statement.setObject(i++, dto.worldGenStep);
statement.setObject(i++, dto.dataType);
statement.setObject(i++, dto.binaryDataFormatVersion);
statement.setObject(i++, dto.dataArray);
core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java → core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java
+24 -9
View File
@@ -17,21 +17,21 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.metaData;
package com.seibel.distanthorizons.core.sql;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.atomic.AtomicLong;
/**
* Contains and represents the meta information ({@link DhSectionPos}, {@link BaseMetaData#dataDetailLevel}, etc.)
* stored at the beginning of files that use the {@link AbstractMetaDataContainerFile}. <Br>
* Which, as of the time of writing, includes: {@link IFullDataSource} and {@link ColumnRenderSource} files.
*/
public class BaseMetaData
/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */
public class DataSourceDto implements IBaseDTO
{
public DhSectionPos pos;
public int checksum;
@@ -46,9 +46,10 @@ public class BaseMetaData
public String dataType;
public byte binaryDataFormatVersion;
public final byte[] dataArray;
public BaseMetaData(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion)
public DataSourceDto(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray)
{
this.pos = pos;
this.checksum = checksum;
@@ -57,6 +58,20 @@ public class BaseMetaData
this.dataType = dataType;
this.binaryDataFormatVersion = binaryDataFormatVersion;
this.dataArray = dataArray;
}
@Override
public String getPrimaryKeyString() { return this.pos.serialize(); }
/** @return a stream for the data contained in this DTO. */
public DhDataInputStream getInputStream() throws IOException
{
InputStream inputStream = new ByteArrayInputStream(this.dataArray);
DhDataInputStream compressedStream = new DhDataInputStream(inputStream);
return compressedStream;
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java
+1 -4
View File
@@ -19,12 +19,9 @@
package com.seibel.distanthorizons.core.sql;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import java.sql.ResultSet;
import java.sql.SQLException;
public class FullDataRepo extends AbstractMetaDataRepo
public class FullDataRepo extends AbstractDataSourceRepo
{
public static final String TABLE_NAME = "DhFullData";
core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java
-57
View File
@@ -1,57 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.sql;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */
public class MetaDataDto implements IBaseDTO
{
public final BaseMetaData baseMetaData;
public final byte[] dataArray;
public MetaDataDto(BaseMetaData baseMetaData, byte[] dataArray)
{
this.baseMetaData = baseMetaData;
this.dataArray = dataArray;
}
@Override
public String getPrimaryKeyString() { return this.baseMetaData.pos.serialize(); }
/** @return a stream for the data contained in this DTO. */
public DhDataInputStream getInputStream() throws IOException
{
InputStream inputStream = new ByteArrayInputStream(this.dataArray);
DhDataInputStream compressedStream = new DhDataInputStream(inputStream);
return compressedStream;
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java
+1 -1
View File
@@ -21,7 +21,7 @@ package com.seibel.distanthorizons.core.sql;
import java.sql.SQLException;
public class RenderDataRepo extends AbstractMetaDataRepo
public class RenderDataRepo extends AbstractDataSourceRepo
{
public static final String TABLE_NAME = "DhRenderData";