diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java index a618a2773..406d64b97 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java @@ -23,7 +23,7 @@ import com.google.common.collect.HashMultimap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.MetaDataDto; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import java.io.IOException; import java.util.*; @@ -120,7 +120,7 @@ public abstract class AbstractFullDataSourceLoader //==============// /** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */ - public IFullDataSource loadTemporaryDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadTemporaryDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException { IFullDataSource dataSource = this.tryGetPooledSource(); if (dataSource != null) @@ -140,9 +140,9 @@ public abstract class AbstractFullDataSourceLoader * * @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down */ - public IFullDataSource loadDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException { - IFullDataSource dataSource = this.createEmptyDataSource(dto.baseMetaData.pos); + IFullDataSource dataSource = this.createEmptyDataSource(dto.pos); dataSource.populateFromStream(dto, dto.getInputStream(), level); return dataSource; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 939e12dfb..1e1a43022 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.MetaDataDto; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -64,6 +64,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu public String getDataTypeName() { return DATA_TYPE_NAME; } private DhSectionPos sectionPos; + private boolean isEmpty = true; public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY; @@ -105,12 +106,12 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetail = inputStream.readInt(); - if (dataDetail != dto.baseMetaData.dataDetailLevel) + if (dataDetail != dto.dataDetailLevel) { - throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetail + " != " + dto.baseMetaData.dataDetailLevel)); + throw new IOException(LodUtil.formatLog("Data level mismatch. Expected: ["+dto.dataDetailLevel+"], found ["+dataDetail+"].")); } int width = inputStream.readInt(); @@ -187,7 +188,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return true; } @Override - public long[][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream dataInputStream) throws IOException + public long[][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream dataInputStream) throws IOException { // Data array length int dataPresentFlag = dataInputStream.readInt(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java index bf48c8216..b239a8c18 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java @@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.MetaDataDto; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -143,15 +143,15 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { - LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() <= MAX_SECTION_DETAIL); + LodUtil.assertTrue(dto.pos.getDetailLevel() > SPARSE_UNIT_DETAIL); + LodUtil.assertTrue(dto.pos.getDetailLevel() <= MAX_SECTION_DETAIL); int dataDetailLevel = inputStream.readShort(); - if (dataDetailLevel != dto.baseMetaData.dataDetailLevel) + if (dataDetailLevel != dto.dataDetailLevel) { - throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.baseMetaData.dataDetailLevel+"]"); + throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.dataDetailLevel+"]"); } // confirm that the detail level is correct @@ -256,11 +256,11 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo return true; } @Override - public long[][][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException + public long[][][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException { // calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize // TODO these values should be constant, should we still be calculating them like this? - int chunks = BitShiftUtil.powerOfTwo(dto.baseMetaData.pos.getDetailLevel() - SPARSE_UNIT_DETAIL); + int chunks = BitShiftUtil.powerOfTwo(dto.pos.getDetailLevel() - SPARSE_UNIT_DETAIL); int dataPointsPerChunk = SECTION_SIZE / chunks; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java index 6422b0e52..a2d188453 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java @@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.MetaDataDto; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -67,8 +67,8 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp @Override public String getDataTypeName() { return DATA_TYPE_NAME; } - private DhSectionPos sectionPos; + private final BitSet isColumnNotEmpty; private boolean isEmpty = true; @@ -120,12 +120,12 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetailLevel = inputStream.readInt(); - if (dataDetailLevel != dto.baseMetaData.dataDetailLevel) + if (dataDetailLevel != dto.dataDetailLevel) { - throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.baseMetaData.dataDetailLevel)); + throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.dataDetailLevel)); } int width = inputStream.readInt(); @@ -188,7 +188,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp return true; } @Override - public StreamDataPointContainer readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException + public StreamDataPointContainer readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException { // is source empty flag int dataPresentFlag = inputStream.readInt(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java index ae098f723..1dcaeed37 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java @@ -19,18 +19,15 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; +import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.IFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.sql.MetaDataDto; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import org.jetbrains.annotations.Nullable; import java.io.IOException; @@ -45,7 +42,7 @@ import java.io.IOException; * @see IIncompleteFullDataSource * @see IStreamableFullDataSource */ -public interface IFullDataSource +public interface IFullDataSource extends IDataSource { /** * This is the byte put between different sections in the binary save file. @@ -57,21 +54,7 @@ public interface IFullDataSource - DhSectionPos getSectionPos(); - - /** - * Returns the name of this data source.
- * Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse - * the binary data when read from file. - */ - String getDataTypeName(); - - /** Returns the detail level of the data contained by this {@link IFullDataSource}. */ - byte getDataDetailLevel(); - /** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */ - byte getDataFormatVersion(); - EDhApiWorldGenerationStep getWorldGenStep(); - + default void update(ChunkSizedFullDataAccessor chunkData, IDhLevel level) { this.update(chunkData); } void update(ChunkSizedFullDataAccessor data); boolean isEmpty(); @@ -110,22 +93,15 @@ public interface IFullDataSource /** * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. * - * @see IStreamableFullDataSource#writeToStream(DhDataOutputStream, IDhLevel) + * @see IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) */ - void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException; + void populateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; /** * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. * - * @see IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) + * @see IStreamableFullDataSource#repopulateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) */ - void populateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; - - /** - * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. - * - * @see IStreamableFullDataSource#repopulateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) - */ - void repopulateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; + void repopulateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java index b48bf785b..4514e5f45 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java @@ -77,7 +77,7 @@ public interface IIncompleteFullDataSource extends IFullDataSource /** * Attempts to convert this {@link IIncompleteFullDataSource} into a {@link CompleteFullDataSource}. * - * @return a new {@link CompleteFullDataSource} if successful, this if the promotion failed, . + * @return a new {@link CompleteFullDataSource} if successful, returns itself if not. */ IFullDataSource tryPromotingToCompleteDataSource(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java index d3bb874e1..e1ce32bef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.MetaDataDto; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; @@ -39,7 +39,7 @@ import java.io.IOException; * * @param defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}. * @param defines the object holding the data points, probably long[][] or long[][][]. - * {@link IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) populateFromStream} + * {@link IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) populateFromStream} * for the full reasoning. */ public interface IStreamableFullDataSource extends IFullDataSource @@ -56,11 +56,11 @@ public interface IStreamableFullDataSource + * Confirms that the given {@link DataSourceDto} is valid for this {@link IStreamableFullDataSource}.
* This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed. * - * @throws IOException if the {@link MetaDataDto} isn't valid for this object. + * @throws IOException if the {@link DataSourceDto} isn't valid for this object. */ - SummaryDataType readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException; + SummaryDataType readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException; void setSourceSummaryData(SummaryDataType summaryData); /** @return true if any data points were present and written, false if this object was empty */ boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException; /** @return null if no data points were present */ - DataContainerType readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException; + DataContainerType readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException; void setDataPoints(DataContainerType dataPoints); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 26228a097..2fd6bcd7a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -22,6 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; +import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; @@ -45,9 +46,9 @@ import java.util.concurrent.atomic.AtomicLong; /** * Stores the render data used to generate OpenGL buffers. * - * @see RenderDataPointUtil + * @see RenderDataPointUtil */ -public class ColumnRenderSource +public class ColumnRenderSource implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -56,7 +57,12 @@ public class ColumnRenderSource public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); public static final byte DATA_FORMAT_VERSION = 1; + @Override + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } + public static final String DATA_NAME = "ColumnRenderSource"; + @Override + public String getDataTypeName() { return DATA_NAME; } /** * This is the byte put between different sections in the binary save file. @@ -81,6 +87,8 @@ public class ColumnRenderSource public AtomicLong localVersion = new AtomicLong(0); // used to track changes to the data source, so that buffers can be updated when necessary + + //==============// // constructors // //==============// @@ -107,7 +115,7 @@ public class ColumnRenderSource * * @throws IOException if the DataInputStream's detail level isn't what was expected */ - public ColumnRenderSource(DhSectionPos sectionPos, ColumnRenderLoader.ParsedColumnData parsedColumnData, IDhLevel level) throws IOException + public ColumnRenderSource(DhSectionPos sectionPos, ColumnRenderSourceLoader.ParsedColumnData parsedColumnData, IDhLevel level) throws IOException { if (sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET != parsedColumnData.detailLevel) { @@ -209,7 +217,9 @@ public class ColumnRenderSource // data update and output // //========================// - public void writeData(DhDataOutputStream outputStream) throws IOException + @Override + public void writeToStream(DhDataOutputStream outputStream, IDhClientLevel level) throws IOException { this.writeToStream(outputStream); } + public void writeToStream(DhDataOutputStream outputStream) throws IOException { outputStream.flush(); @@ -302,11 +312,8 @@ public class ColumnRenderSource } } - /** - * Doesn't write anything to file. - * @return true if any data was changed, false otherwise - */ - public boolean updateWithChunkData(ChunkSizedFullDataAccessor chunkDataView, IDhClientLevel level) + @Override + public void update(ChunkSizedFullDataAccessor chunkDataView, IDhClientLevel level) { final String errorMessagePrefix = "Unable to complete fastWrite for RenderSource pos: [" + this.sectionPos + "] and chunk pos: [" + chunkDataView.chunkPos + "]. Error:"; @@ -333,14 +340,14 @@ public class ColumnRenderSource || blockOffsetZ + LodUtil.CHUNK_WIDTH > this.getWidthInDataPoints()) { LOGGER.warn(errorMessagePrefix+"Data offset is out of bounds."); - return false; + return; } if (Thread.interrupted()) { LOGGER.warn(errorMessagePrefix+"write interrupted."); - return false; + return; } @@ -399,7 +406,7 @@ public class ColumnRenderSource int chunksPerColumn = sourceStartingChangePos.getWidthAtDetail(chunkDataView.getSectionPos().getDetailLevel()); if (chunkDataView.getSectionPos().getX() % chunksPerColumn != 0 || chunkDataView.getSectionPos().getZ() % chunksPerColumn != 0) { - return false; // not a multiple of the column size, so no change + return; // not a multiple of the column size, so no change } int relStartX = Math.floorMod(sourceStartingChangePos.x, this.getWidthInDataPoints()); int relStartZ = Math.floorMod(sourceStartingChangePos.z, this.getWidthInDataPoints()); @@ -418,8 +425,6 @@ public class ColumnRenderSource { this.localVersion.incrementAndGet(); } - - return dataChanged; } @@ -442,9 +447,14 @@ public class ColumnRenderSource public long getRoughRamUsageInBytes() { return (long) this.renderDataContainer.length * Long.BYTES; } public DhSectionPos getSectionPos() { return this.sectionPos; } + @Override + public String getPrimaryKeyString() { return this.sectionPos.serialize(); } public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } + @Override + public EDhApiWorldGenerationStep getWorldGenStep() { return EDhApiWorldGenerationStep.EMPTY; } + /** @return how many data points wide this {@link ColumnRenderSource} is. */ public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); } public byte getDetailOffset() { return SECTION_SIZE_OFFSET; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java similarity index 87% rename from core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderLoader.java rename to core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java index 45d34b8cf..26e72a894 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java @@ -20,9 +20,9 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.file.renderfile.RenderDataMetaFile; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; @@ -31,26 +31,26 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; /** - * Handles loading and parsing {@link RenderDataMetaFile}s to create {@link ColumnRenderSource}s.

+ * Handles loading and parsing {@link DataSourceDto}s to create {@link ColumnRenderSource}s.

* - * Please see the {@link ColumnRenderLoader#loadRenderSource} method to see what + * Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what * file versions this class can handle. */ -public class ColumnRenderLoader +public class ColumnRenderSourceLoader { - public static ColumnRenderLoader INSTANCE = new ColumnRenderLoader(); + public static ColumnRenderSourceLoader INSTANCE = new ColumnRenderSourceLoader(); private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private ColumnRenderLoader() { } + private ColumnRenderSourceLoader() { } - public ColumnRenderSource loadRenderSource(RenderDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException + public ColumnRenderSource loadRenderSource(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { - int dataFileVersion = dataFile.baseMetaData.binaryDataFormatVersion; + int dataFileVersion = dto.binaryDataFormatVersion; switch (dataFileVersion) { @@ -60,10 +60,10 @@ public class ColumnRenderLoader ParsedColumnData parsedColumnData = readDataV1(inputStream, level.getMinY()); if (parsedColumnData.isEmpty) { - LOGGER.warn("Empty render file " + dataFile.pos); + LOGGER.warn("Empty render file " + dto.pos); } - return new ColumnRenderSource(dataFile.pos, parsedColumnData, level); + return new ColumnRenderSource(dto.pos, parsedColumnData, level); default: throw new IOException("Invalid Data: The data version [" + dataFileVersion + "] is not supported"); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java new file mode 100644 index 000000000..7333b4228 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -0,0 +1,364 @@ +package com.seibel.distanthorizons.core.file; + +import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.*; +import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.ClosedChannelException; +import java.util.Enumeration; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.zip.Adler32; +import java.util.zip.CheckedOutputStream; + +public abstract class AbstractDataSourceHandler, TDhLevel extends IDhLevel> implements ISourceProvider +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final Timer DELAYED_SAVE_TIMER = new Timer(); + /** How long a data source must remain un-modified before being written to disk. */ + private static final int SAVE_DELAY_IN_MS = 4_000; + + /** + * The highest numerical detail level known about. + * Used when determining which positions to update. + */ + protected final AtomicInteger topSectionDetailLevelRef; + protected final int minDetailLevel = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + + protected final ConcurrentHashMap unsavedDataSourceBySectionPos = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap saveTimerTasksBySectionPos = new ConcurrentHashMap<>(); + protected final ReentrantLock[] updateLockArray; + + protected final TDhLevel level; + protected final File saveDir; + + public final AbstractDataSourceRepo repo; + + + + //=============// + // constructor // + //=============// + + public AbstractDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, AbstractDataSourceRepo repo) { this(level, saveStructure, repo, null); } + public AbstractDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, AbstractDataSourceRepo repo, @Nullable File saveDirOverride) + { + this.level = level; + this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; + if (!this.saveDir.exists() && !this.saveDir.mkdirs()) + { + LOGGER.warn("Unable to create full data folder, file saving may fail."); + } + + // the lock array's length is double the number of CPU cores so the number of collisions + // should be relatively low without having too many extra locks + this.updateLockArray = new ReentrantLock[Runtime.getRuntime().availableProcessors() * 2]; + for (int i = 0; i < this.updateLockArray.length; i++) + { + this.updateLockArray[i] = new ReentrantLock(); + } + + this.repo = repo; + + // determine the top detail level currently in the database + int maxSectionDetailLevel = this.repo.getMaxSectionDetailLevel(); + this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel); + } + + + + + //==================// + // abstract methods // + //==================// + + protected abstract TDataSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException; + /** + * Creates a new data source using any DTOs already present in the database. + * Can return null if there was an issue, but in general should return at least an empty data source. + */ + @Nullable + protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos); + + protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); + + + + //==============// + // data reading // + //==============// + + /** + * Returns the {@link TDataSource} for the given section position.
+ * The returned data source may be null if there was a problem.

+ * + * This call is concurrent. I.e. it supports being called by multiple threads at the same time. + */ + @Override + public CompletableFuture getAsync(DhSectionPos pos) + { + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + /** + * Should only be used in internal file handler methods where we are already running on a file handler thread. + * Can return null if there was a problem. + * @see AbstractDataSourceHandler#getAsync(DhSectionPos) + */ + @Nullable + public TDataSource get(DhSectionPos pos) + { + // used the unsaved data source if present + if (this.unsavedDataSourceBySectionPos.containsKey(pos)) + { + return this.unsavedDataSourceBySectionPos.get(pos); + } + // an unsaved data source isn't present + // check the database + + + // increase the top detail level if necessary + this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); + + + TDataSource dataSource = null; + try + { + DataSourceDto dto = this.repo.getByPrimaryKey(pos.serialize()); + if (dto != null) + { + // load from file + dataSource = this.createDataSourceFromDto(dto); + } + else + { + // attempt to create from any existing files + dataSource = this.createNewDataSourceFromExistingDtos(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); + } + + return dataSource; + } + + + + //===============// + // data updating // + //===============// + + @Override + public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView) + { + DhSectionPos chunkSectionPos = chunkDataView.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + this.recursivelyUpdateDataSourcesAsync(chunkSectionPos, chunkDataView); + } + /** Updates every data source from this position up to {@link AbstractDataSourceHandler#topSectionDetailLevelRef} */ + protected void recursivelyUpdateDataSourcesAsync(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView) + { + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return; + } + + // update up until we reach the highest available data source + if (pos.getDetailLevel() > this.topSectionDetailLevelRef.get()) + { + return; + } + + + executor.execute(() -> + { + DhSectionPos chunkSectionPos = chunkDataView.getSectionPos(); + LodUtil.assertTrue(chunkSectionPos.overlapsExactly(pos), "Update failed, chunk [" + chunkSectionPos + "] does not overlap section [" + pos + "]."); + + // update this pos + this.updateDataSourceAtPos(pos, chunkDataView); + + // recursively update the parent pos + DhSectionPos parentPos = pos.getParentPos(); + this.recursivelyUpdateDataSourcesAsync(parentPos, chunkDataView); + + }); + } + protected void updateDataSourceAtPos(DhSectionPos pos, ChunkSizedFullDataAccessor chunkData) + { + // a lock is necessary to prevent two threads from writing to the same position at once, + // if that happens only the second update will apply and the LOD will end up with hole(s) + ReentrantLock updateLock = this.getUpdateLockForPos(pos); + + try + { + updateLock.lock(); + + // get or create the data source + TDataSource dataSource = this.get(pos); + if (dataSource == null) + { + dataSource = this.makeEmptyDataSource(pos); + } + dataSource.update(chunkData, this.level); + + this.queueDelayedSave(dataSource); + } + catch (Exception e) + { + LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); + } + finally + { + updateLock.unlock(); + } + } + /** + * Queues the given data source to save after {@link AbstractDataSourceHandler#SAVE_DELAY_IN_MS} + * milliseconds have passed without any additional modifications.

+ * + * This prevents repeatedly reading/writing the same data source to/from disk if said + * source is currently being updated via world gen or chunk modifications. + * This drastically reduces disk usage and improves performance. + */ + protected void queueDelayedSave(TDataSource dataSource) + { + DhSectionPos pos = dataSource.getSectionPos(); + + // put the data source in memory until it can be flushed to disk + this.unsavedDataSourceBySectionPos.put(pos, dataSource); + + TimerTask task = new TimerTask() + { + @Override + public void run() + { + try + { + final TDataSource finalDataSource = AbstractDataSourceHandler.this.unsavedDataSourceBySectionPos.remove(pos); + + // this can rarely happen due to imperfect concurrency handling, + // if the data source is null that just means it has already been saved so nothing needs to be done + if (finalDataSource != null) + { + AbstractDataSourceHandler.this.writeDataSourceToFile(finalDataSource); + } + } + catch (ClosedByInterruptException e) // thrown by buffers that are interrupted + { + // expected if the file handler is shut down, the exception can be ignored + } + catch (IOException e) + { + LOGGER.error("Failed to save updated data for section " + pos, e); + } + } + }; + DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS); + + // cancel the old save timer if present + // (this is equivalent to restarting the timer) + TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task); + if (oldTask != null) + { + oldTask.cancel(); + } + } + protected void writeDataSourceToFile(TDataSource dataSource) throws IOException + { + LodUtil.assertTrue(dataSource != null); + + try + { + // write the outputs to a stream to prep for writing to the database + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + + // the order of these streams is important, otherwise the checksum won't be calculated + CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); + // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, + // but since this stream will be closed immediately after writing anyway, it won't be an issue + DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut); + + dataSource.writeToStream(compressedOut, AbstractDataSourceHandler.this.level); + + compressedOut.flush(); + int checksum = (int) checkedOut.getChecksum().getValue(); + byteArrayOutputStream.close(); + + + // save the DTO + DataSourceDto newDto = new DataSourceDto( + dataSource.getSectionPos(), checksum, + dataSource.getDataDetailLevel(), dataSource.getWorldGenStep(), dataSource.getDataTypeName(), + dataSource.getDataFormatVersion(), + byteArrayOutputStream.toByteArray()); + this.repo.save(newDto); + } + catch (ClosedChannelException e) // includes ClosedByInterruptException + { + // expected if the file handler is shut down, the exception can be ignored + } + } + + + + //================// + // helper methods // + //================// + + /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ + protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } + + + + //=========// + // cleanup // + //=========// + + @Override + public void close() + { + LOGGER.info("Closing ["+this.getClass().getSimpleName()+"] for level: ["+this.level+"], saving ["+this.saveTimerTasksBySectionPos.size()+"] positions."); + + Enumeration list = this.saveTimerTasksBySectionPos.keys(); + while (list.hasMoreElements()) + { + DhSectionPos pos = list.nextElement(); + TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos); + if (saveTask != null) + { + saveTask.run(); + saveTask.cancel(); + } + } + + LOGGER.info("["+this.getClass().getSimpleName()+"] saving complete, closing repo."); + this.repo.close(); + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java new file mode 100644 index 000000000..a4c19974e --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -0,0 +1,54 @@ +package com.seibel.distanthorizons.core.file; + +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.IBaseDTO; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; + +import java.io.IOException; + +/** + * Base for all data sources. + * + * @param what type of level this data source can be created from + */ +public interface IDataSource extends IBaseDTO +{ + + DhSectionPos getSectionPos(); + @Override + default String getPrimaryKeyString() { return this.getSectionPos().serialize(); } + + + + //===============// + // file handling // + //===============// + + void update(ChunkSizedFullDataAccessor chunkData, TDhLevel level); + + void writeToStream(DhDataOutputStream outputStream, TDhLevel level) throws IOException; + + + + //===========// + // meta data // + //===========// + + /** Returns the detail level of the data contained by this {@link IFullDataSource}. */ + byte getDataDetailLevel(); + EDhApiWorldGenerationStep getWorldGenStep(); + /** + * Returns the name of this data source.
+ * Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse + * the binary data when read from file. + */ + String getDataTypeName(); + /** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */ + byte getDataFormatVersion(); + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java new file mode 100644 index 000000000..9d27be3af --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java @@ -0,0 +1,15 @@ +package com.seibel.distanthorizons.core.file; + +import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.pos.DhSectionPos; + +import java.util.concurrent.CompletableFuture; + +public interface ISourceProvider, TDhLevel extends IDhLevel> extends AutoCloseable +{ + CompletableFuture getAsync(DhSectionPos pos); + + void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData); + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 75347c8cd..7caea1a5d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -20,66 +20,31 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; -import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; +import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.sql.FullDataRepo; -import com.seibel.distanthorizons.core.sql.MetaDataDto; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.awt.*; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.nio.channels.ClosedByInterruptException; -import java.nio.channels.ClosedChannelException; import java.sql.SQLException; import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; -import java.util.zip.Adler32; -import java.util.zip.CheckedOutputStream; -public class FullDataFileHandler implements IFullDataSourceProvider +public class FullDataFileHandler extends AbstractDataSourceHandler implements IFullDataSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final Timer DELAYED_SAVE_TIMER = new Timer(); - /** How long a data source must remain un-modified before being written to disk. */ - private static final int SAVE_DELAY_IN_MS = 4_000; - - protected final ConcurrentHashMap unsavedDataSourceBySectionPos = new ConcurrentHashMap<>(); - protected final ConcurrentHashMap saveTimerTasksBySectionPos = new ConcurrentHashMap<>(); - protected final ReentrantLock[] updateLockArray; - - protected final IDhLevel level; - protected final File saveDir; - - /** - * The highest numerical detail level known about. - * Used when determining which positions to update. - */ - protected final AtomicInteger topSectionDetailLevelRef; - protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; - - public final FullDataRepo fullDataRepo; - @Override - public FullDataRepo getRepo() { return this.fullDataRepo; } @@ -90,25 +55,15 @@ public class FullDataFileHandler implements IFullDataSourceProvider public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { - this.level = level; - this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; - if (!this.saveDir.exists() && !this.saveDir.mkdirs()) - { - LOGGER.warn("Unable to create full data folder, file saving may fail."); - } - - // the lock array's length is double the number of CPU cores so the number of collisions - // should be relatively low without having too many extra locks - this.updateLockArray = new ReentrantLock[Runtime.getRuntime().availableProcessors() * 2]; - for (int i = 0; i < this.updateLockArray.length; i++) - { - this.updateLockArray[i] = new ReentrantLock(); - } - + super(level, saveStructure, createRepo(level, saveStructure), saveDirOverride); + } + private static FullDataRepo createRepo(IDhLevel level, AbstractSaveStructure saveStructure) + { + File saveDir = saveStructure.getFullDataFolder(level.getLevelWrapper()); try { - this.fullDataRepo = new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + return new FullDataRepo("jdbc:sqlite", saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); } catch (SQLException e) { @@ -116,81 +71,23 @@ public class FullDataFileHandler implements IFullDataSourceProvider // or the database update failed throw new RuntimeException(e); } - - // determine the top detail level currently in the database - int maxSectionDetailLevel = this.fullDataRepo.getMaxSectionDetailLevel(); - this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel); } - //==============// - // data reading // - //==============// + //====================// + // Abstract overrides // + //====================// - /** - * Returns the {@link IFullDataSource} for the given section position.
- * The returned data source may be null.

- * - * For now, if result is null, it prob means error has occurred when loading or creating the file object.

- * - * This call is concurrent. I.e. it supports being called by multiple threads at the same time. - */ @Override - public CompletableFuture getAsync(DhSectionPos pos) + protected IFullDataSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException { - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return CompletableFuture.completedFuture(null); - } - - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); - } - /** - * Should be used in internal methods where we are already running on a file handler thread. - * @see FullDataFileHandler#getAsync(DhSectionPos) - */ - protected IFullDataSource get(DhSectionPos pos) - { - // used the unsaved data source if present - if (this.unsavedDataSourceBySectionPos.containsKey(pos)) - { - return this.unsavedDataSourceBySectionPos.get(pos); - } - // an unsaved data source isn't present - // check the database - - - // increase the top detail level if necessary - this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - - - IFullDataSource dataSource = null; - try - { - MetaDataDto dto = this.fullDataRepo.getByPrimaryKey(pos.serialize()); - if (dto != null) - { - // load from file - AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.baseMetaData.dataType, dto.baseMetaData.binaryDataFormatVersion); - dataSource = loader.loadDataSource(dto, this.level); - } - else - { - // attempt to create from any existing files - dataSource = this.createNewDataSourceFromExistingDtos(pos); - } - } - catch (InterruptedException ignore) { } - catch (IOException e) - { - LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); - } - + AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.dataType, dto.binaryDataFormatVersion); + IFullDataSource dataSource = loader.loadDataSource(dto, this.level); return dataSource; } /** Creates a new data source using any DTOs already present in the database. */ + @Override protected IFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) { IIncompleteFullDataSource newFullDataSource = this.makeEmptyDataSource(pos); @@ -205,7 +102,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider } - // TODO replace with a SQL query, it should be much faster + // get all non-empty sections to sample from ArrayList samplePosList = new ArrayList<>(); ArrayList possibleChildList = new ArrayList<>(); pos.forEachChild((childPos) -> @@ -218,7 +115,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider while (possibleChildList.size() != 0) { DhSectionPos possiblePos = possibleChildList.remove(possibleChildList.size()-1); - if (this.fullDataRepo.existsWithPrimaryKey(possiblePos.serialize())) + if (this.repo.existsWithPrimaryKey(possiblePos.serialize())) { samplePosList.add(possiblePos); } @@ -268,200 +165,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider return newFullDataSource.tryPromotingToCompleteDataSource(); } - - - //===============// - // data updating // - //===============// - @Override - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView) - { - DhSectionPos chunkSectionPos = chunkDataView.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); - this.recursivelyUpdateDataSourcesAsync(chunkSectionPos, chunkDataView); - } - /** Updates every data source from this position up to {@link FullDataFileHandler#topSectionDetailLevelRef} */ - private void recursivelyUpdateDataSourcesAsync(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView) - { - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return; - } - - // update up until we reach the highest available data source - if (pos.getDetailLevel() > this.topSectionDetailLevelRef.get()) - { - return; - } - - - executor.execute(() -> - { - DhSectionPos chunkSectionPos = chunkDataView.getSectionPos(); - LodUtil.assertTrue(chunkSectionPos.overlapsExactly(pos), "Update failed, chunk [" + chunkSectionPos + "] does not overlap section [" + pos + "]."); - - // update this pos - this.updateDataSourceAtPos(pos, chunkDataView); - - // recursively update the parent pos - DhSectionPos parentPos = pos.getParentPos(); - this.recursivelyUpdateDataSourcesAsync(parentPos, chunkDataView); - - }); - } - private void updateDataSourceAtPos(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView) - { - // a lock is necessary to prevent two threads from writing to the same position at once, - // if that happens only the second update will apply and the LOD will end up with hole(s) - ReentrantLock updateLock = this.getUpdateLockForPos(pos); - - try - { - updateLock.lock(); - - // get or create the data source - IFullDataSource fullDataSource = this.get(pos); - if (fullDataSource == null) - { - fullDataSource = this.makeEmptyDataSource(pos); - } - fullDataSource.update(chunkDataView); - - // try promoting the datasource - if (fullDataSource instanceof IIncompleteFullDataSource) - { - IIncompleteFullDataSource incompleteFullDataSource = (IIncompleteFullDataSource) fullDataSource; - fullDataSource = incompleteFullDataSource.tryPromotingToCompleteDataSource(); - } - - this.queueDelayedSave(fullDataSource); - } - catch (Exception e) - { - LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); - } - finally - { - updateLock.unlock(); - } - } - /** - * Queues the given data source to save after {@link FullDataFileHandler#SAVE_DELAY_IN_MS} - * milliseconds have passed without any additional modifications.

- * - * This prevents repeatedly reading/writing the same data source to/from disk if said - * source is currently being updated via world gen or chunk modifications. - * This drastically reduces disk usage and improves performance. - */ - private void queueDelayedSave(IFullDataSource fullDataSource) - { - DhSectionPos pos = fullDataSource.getSectionPos(); - - // put the data source in memory until it can be flushed to disk - this.unsavedDataSourceBySectionPos.put(pos, fullDataSource); - - TimerTask task = new TimerTask() - { - @Override - public void run() - { - try - { - final IFullDataSource finalFullDataSource = FullDataFileHandler.this.unsavedDataSourceBySectionPos.remove(pos); - - // this can rarely happen due to imperfect concurrency handling, - // if the data source is null that just means it has already been saved so nothing needs to be done - if (finalFullDataSource != null) - { - FullDataFileHandler.this.writeDataSourceToFile(finalFullDataSource, (bufferedOutputStream) -> - { - try - { - finalFullDataSource.writeToStream(bufferedOutputStream, FullDataFileHandler.this.level); - } - catch (Exception e) - { - // if this try catch isn't included an empty exception will be thrown instead, which makes debugging extremely painful - LOGGER.error("Error writing data stream for pos: [" + finalFullDataSource.getSectionPos() + "], error: " + e.getMessage(), e); - } - }); - } - } - catch (ClosedByInterruptException e) // thrown by buffers that are interrupted - { - // expected if the file handler is shut down, the exception can be ignored - //LOGGER.warn("FullData file writing interrupted.", e); - } - catch (IOException e) - { - LOGGER.error("Failed to save updated data for section " + pos, e); - } - } - }; - DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS); - - // - TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task); - if (oldTask != null) - { - oldTask.cancel(); - } - } - private void writeDataSourceToFile(IFullDataSource fullDataSource, AbstractMetaDataContainerFile.IMetaDataWriterFunc dataWriterFunc) throws IOException - { - LodUtil.assertTrue(fullDataSource != null); - - - boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); - if (showFullDataFileStatus) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW), - 0.2, 16f)); - } - - try - { - // write the outputs to a stream to prep for writing to the database - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - - // the order of these streams is important, otherwise the checksum won't be calculated - CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); - // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, - // but since this stream will be closed immediately after writing anyway, it won't be an issue - DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut); - - dataWriterFunc.writeBinaryDataToStream(compressedOut); - - compressedOut.flush(); - int checksum = (int) checkedOut.getChecksum().getValue(); - byteArrayOutputStream.close(); - - - // save the DTO - BaseMetaData baseMetaData = new BaseMetaData(fullDataSource.getSectionPos(), checksum, - fullDataSource.getDataDetailLevel(), fullDataSource.getWorldGenStep(), fullDataSource.getDataTypeName(), - fullDataSource.getDataFormatVersion()); - MetaDataDto newDto = new MetaDataDto(baseMetaData, byteArrayOutputStream.toByteArray()); - this.fullDataRepo.save(newDto); - } - catch (ClosedChannelException e) // includes ClosedByInterruptException - { - // expected if the file handler is shut down, the exception can be ignored - //LOGGER.warn(AbstractMetaDataContainerFile.class.getSimpleName()+" file writing interrupted. Error: "+e.getMessage()); - } - } - - - - //================// - // helper methods // - //================// - - /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ - protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } - protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { return pos.getDetailLevel() <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL ? @@ -471,29 +175,33 @@ public class FullDataFileHandler implements IFullDataSourceProvider - //=========// - // cleanup // - //=========// + //===================// + // extension methods // + //===================// @Override - public void close() + public void writeDataSourceToFile(IFullDataSource fullDataSource) throws IOException { - LOGGER.info("Closing file handler for level: ["+this.level+"], saving ["+this.saveTimerTasksBySectionPos.size()+"] positions."); - - Enumeration list = this.saveTimerTasksBySectionPos.keys(); - while (list.hasMoreElements()) + // doing this here guarantees that all changes are caught and promoted + if (fullDataSource instanceof IIncompleteFullDataSource) { - DhSectionPos pos = list.nextElement(); - TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos); - if (saveTask != null) - { - saveTask.run(); - saveTask.cancel(); - } + fullDataSource = ((IIncompleteFullDataSource) fullDataSource).tryPromotingToCompleteDataSource(); } - LOGGER.info("File handler saving complete, closing repo."); - this.fullDataRepo.close(); + super.writeDataSourceToFile(fullDataSource); + + // save has completed + boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); + if (showFullDataFileStatus) + { + DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( + new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW), + 0.2, 16f)); + } } + @Override + public int getUnsavedDataSourceCount() { return this.unsavedDataSourceBySectionPos.size(); } + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 7ec6d8872..ee94e10dd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -66,7 +66,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler //===========// @Override - protected IFullDataSource get(DhSectionPos pos) + public IFullDataSource get(DhSectionPos pos) { IFullDataSource dataSource = super.get(pos); @@ -256,7 +256,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler { if (chunkSizedFullDataSource.getSectionPos().overlapsExactly(this.loadedTargetFullDataSource.getSectionPos())) { - ((DhLevel) level).saveWrites(chunkSizedFullDataSource); + ((DhLevel) level).updateDataSourcesWithChunkData(chunkSizedFullDataSource); //GeneratedFullDataFileHandler.this.write(this.loadedTargetFullDataSource.getSectionPos(), chunkSizedFullDataSource); } }; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index aa7a29563..5d52af2e5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -21,6 +21,10 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.file.ISourceProvider; +import com.seibel.distanthorizons.core.level.IDhClientLevel; +import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.FullDataRepo; @@ -30,12 +34,13 @@ import java.util.concurrent.CompletableFuture; * Handles reading, writing, and updating {@link IFullDataSource}'s.
* Should be backed by a database handled by a {@link FullDataRepo}. */ -public interface IFullDataSourceProvider extends AutoCloseable +public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable { CompletableFuture getAsync(DhSectionPos pos); + IFullDataSource get(DhSectionPos pos); void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData); - FullDataRepo getRepo(); + int getUnsavedDataSourceCount(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java deleted file mode 100644 index f383b5d41..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.metaData; - -import java.io.*; -import java.nio.channels.ClosedChannelException; -import java.nio.file.*; -import java.util.zip.Adler32; -import java.util.zip.CheckedOutputStream; - -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.AbstractDhRepo; -import com.seibel.distanthorizons.core.sql.MetaDataDto; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import org.apache.logging.log4j.Logger; - -/** - * This represents the data appended to any file we write.
- * Contains a {@link BaseMetaData} which holds most of the necessary values written to the file.

- * - * Used size: 40 bytes
- * Remaining space: 24 bytes
- * Total size: 64 bytes


- * - * - * Metadata format:

- * - * 4 bytes: metadata identifier bytes: "DHv0" (in ascii: 0x44 48 76 30) this signals the file is in the metadata format
- * 4 bytes: section X position
- * 4 bytes: section Y position (Unused, for future proofing)
- * 4 bytes: section Z position

- * - * 4 bytes: data checksum
- * 1 byte: section detail level
- * 1 byte: data detail level // Note: not sure if this is needed
- * 1 byte: loader version
- * 1 byte: unused

- * - * 8 bytes: datatype identifier

- * - * 8 bytes: data version - * - */ -public abstract class AbstractMetaDataContainerFile -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - - /** - * Will be null if no file exists for this object.
- * NOTE: Only use {@link BaseMetaData#pos} when initially setting up this object, afterwards the standalone {@link AbstractMetaDataContainerFile#pos} should be used. - */ - public volatile BaseMetaData baseMetaData = null; - - /** Should be used instead of the position inside {@link AbstractMetaDataContainerFile#baseMetaData} */ - public final DhSectionPos pos; - - - - //==============// - // constructors // - //==============// - - /** - * Create a metaFile in this path. - * - * @throws FileAlreadyExistsException If the path already has a file. - */ - protected AbstractMetaDataContainerFile(DhSectionPos pos) { this.pos = pos; } - - /** - * Creates an {@link AbstractMetaDataContainerFile} with the file at the given path. - * - * @throws IOException if the file was formatted incorrectly - * @throws FileNotFoundException if no file exists for the given path - */ - protected AbstractMetaDataContainerFile(BaseMetaData baseMetaData) throws IOException - { - this.baseMetaData = baseMetaData; - this.pos = this.baseMetaData.pos; - } - - - //==============// - // file writing // - //==============// - - public void writeToDatabase(IMetaDataWriterFunc dataWriterFunc, AbstractDhRepo repo) throws IOException - { - LodUtil.assertTrue(this.baseMetaData != null); - - try - { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - - // the order of these streams is important, otherwise the checksum won't be calculated - CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); - // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, but since this stream will be closed immediately after writing anyway, it won't be an issue - DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut); - - // write the contained data - dataWriterFunc.writeBinaryDataToStream(compressedOut); - compressedOut.flush(); - this.baseMetaData.checksum = (int) checkedOut.getChecksum().getValue(); - - - byteArrayOutputStream.close(); - - - MetaDataDto dto = new MetaDataDto(this.baseMetaData, byteArrayOutputStream.toByteArray()); - repo.save(dto); - } - catch (ClosedChannelException e) // includes ClosedByInterruptException - { - // expected if the file handler is shut down, the exception can be ignored - //LOGGER.warn(AbstractMetaDataContainerFile.class.getSimpleName()+" file writing interrupted. Error: "+e.getMessage()); - } - } - - - - //================// - // helper classes // - //================// - - /** TODO replace with a method that accepts a {@link DhDataOutputStream} and writes to that instead */ - @FunctionalInterface - public interface IMetaDataWriterFunc { void writeBinaryDataToStream(T t) throws IOException; } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java index fcc10c2c1..61050bf73 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java @@ -20,29 +20,27 @@ package com.seibel.distanthorizons.core.file.renderfile; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.file.ISourceProvider; +import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.sql.FullDataRepo; import com.seibel.distanthorizons.core.sql.RenderDataRepo; import java.util.concurrent.CompletableFuture; /** - * This represents LOD data that is stored in long term storage (IE LOD files stored on the hard drive)
- * Example: {@link RenderSourceFileHandler RenderSourceFileHandler}

- * - * This is used to create {@link ColumnRenderSource}'s + * Handles reading, writing, and updating {@link ColumnRenderSource}'s.
+ * Should be backed by a database handled by a {@link RenderDataRepo}. */ -public interface IRenderSourceProvider extends AutoCloseable +public interface IRenderSourceProvider extends ISourceProvider { - CompletableFuture readAsync(DhSectionPos pos); + CompletableFuture getAsync(DhSectionPos pos); - void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData); - CompletableFuture flushAndSaveAsync(); + void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData); /** Deletes any data stored in the render cache so it can be re-created */ void deleteRenderCache(); - - RenderDataRepo getRepo(); - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java deleted file mode 100644 index 061889f51..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java +++ /dev/null @@ -1,493 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.renderfile; - -import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; -import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderLoader; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.sql.MetaDataDto; -import com.seibel.distanthorizons.core.util.AtomicsUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; - -import java.awt.*; -import java.io.*; -import java.lang.ref.SoftReference; -import java.util.Random; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicReference; - -/** Represents a File that contains a {@link ColumnRenderSource}. */ -public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - public static final boolean ALWAYS_INVALIDATE_CACHE = false; - public static final String RENDER_SOURCE_TYPE = ColumnRenderSource.DATA_NAME; - - - /** - * Can be cleared if the garbage collector determines there isn't enough space.

- * - * When clearing, don't set to null, instead create a SoftReference containing null. - * This makes null checks simpler. - */ - private SoftReference cachedRenderDataSourceRef = new SoftReference<>(null); - private final AtomicReference> renderSourceLoadFutureRef = new AtomicReference<>(null); - - private final IDhClientLevel clientLevel; - private final IFullDataSourceProvider fullDataSourceProvider; - private final IRenderSourceProvider renderDataSourceProvider; - private boolean doesDtoExist; - - - - //=============// - // constructor // - //=============// - - /** - * NOTE: should only be used if there is NOT an existing file. - * @throws IOException if a file already exists for this position - */ - public static RenderDataMetaFile createNewFileForPos(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, DhSectionPos pos) throws IOException { return new RenderDataMetaFile(fullDataSourceProvider, renderDataSourceProvider, clientLevel, pos); } - private RenderDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, DhSectionPos pos) throws IOException - { - super(pos); - this.fullDataSourceProvider = fullDataSourceProvider; - this.renderDataSourceProvider = renderDataSourceProvider; - this.clientLevel = clientLevel; - LodUtil.assertTrue(this.baseMetaData == null); - this.doesDtoExist = false; - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus); - } - - - /** - * NOTE: should only be used if there IS an existing file. - * @throws IOException if no file exists for this position - */ - public static RenderDataMetaFile createFromExistingFile(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, MetaDataDto metaDataDto) throws IOException { return new RenderDataMetaFile(fullDataSourceProvider, renderDataSourceProvider, clientLevel, metaDataDto); } - private RenderDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IRenderSourceProvider renderDataSourceProvider, IDhClientLevel clientLevel, MetaDataDto metaDataDto) throws IOException - { - super(metaDataDto.baseMetaData); - this.fullDataSourceProvider = fullDataSourceProvider; - this.renderDataSourceProvider = renderDataSourceProvider; - this.clientLevel = clientLevel; - LodUtil.assertTrue(this.baseMetaData != null); - this.doesDtoExist = true; - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus); - } - - - - //=============// - // data update // - //=============// - - public void updateChunkIfSourceExistsAsync(ChunkSizedFullDataAccessor chunkDataView) - { - DhSectionPos chunkSectionPos = chunkDataView.getSectionPos(); - LodUtil.assertTrue(this.pos.overlapsExactly(chunkSectionPos), "Chunk pos " + chunkSectionPos + " doesn't overlap with section " + this.pos); - - // update the render source if one exists - CompletableFuture renderSourceLoadFuture = this.getCachedDataSourceAsync(true); - if (renderSourceLoadFuture == null) - { - return; - } - - - renderSourceLoadFuture.thenAccept((renderSource) -> - { - boolean dataUpdated = renderSource.updateWithChunkData(chunkDataView, this.clientLevel); - - - // add a debug particle - boolean showRenderDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus.get(); - if (showRenderDataFileStatus) - { - float offset = new Random(System.nanoTime() ^ Thread.currentThread().getId()).nextFloat() * 16f; - Color debugColor = dataUpdated ? Color.blue : Color.red; - DebugRenderer.makeParticle( - new DebugRenderer.BoxParticle( - new DebugRenderer.Box(chunkDataView.getSectionPos(), 32f, 64f + offset, 0.07f, debugColor), - 2.0, 16f - ) - ); - } - }); - } - - - - //======================// - // render source getter // - //======================// - - public CompletableFuture getOrLoadCachedDataSourceAsync(Executor fileReaderThreads) - { - CompletableFuture renderSourceLoadFuture = this.getCachedDataSourceAsync(true); - if (renderSourceLoadFuture != null) - { - // return the in-process future - return renderSourceLoadFuture; - } - else - { - // there is no cached data, we'll have to load it - - renderSourceLoadFuture = new CompletableFuture<>(); - if (!this.renderSourceLoadFutureRef.compareAndSet(null, renderSourceLoadFuture)) - { - // two threads attempted to start this job at the same time, only use the first future - renderSourceLoadFuture = this.renderSourceLoadFutureRef.get(); - } - } - - - - final CompletableFuture getSourceFuture = renderSourceLoadFuture; - if (!this.doesDtoExist) - { - // create a new Meta file and render source - - - // create an empty render source - byte dataDetailLevel = (byte) (this.pos.getDetailLevel() - ColumnRenderSource.SECTION_SIZE_OFFSET); - int verticalSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(dataDetailLevel); - ColumnRenderSource newColumnRenderSource = new ColumnRenderSource(this.pos, verticalSize, this.clientLevel.getMinY()); - - this.baseMetaData = new BaseMetaData( - newColumnRenderSource.getSectionPos(), -1, newColumnRenderSource.getDataDetailLevel(), - newColumnRenderSource.worldGenStep, RENDER_SOURCE_TYPE, - newColumnRenderSource.getRenderDataFormatVersion()); - - this.updateRenderCacheAsync(newColumnRenderSource).whenComplete((voidObj, ex) -> - { - this.cachedRenderDataSourceRef = new SoftReference<>(newColumnRenderSource); - - this.renderSourceLoadFutureRef.set(null); - getSourceFuture.complete(newColumnRenderSource); - }); - } - else - { - // load the existing Meta file and render source - - CompletableFuture.supplyAsync(() -> - { - if (this.baseMetaData == null) - { - throw new IllegalStateException("Meta data not loaded!"); - } - - // Load the render source file. - ColumnRenderSource renderSource; - try (InputStream inputStream = this.getInputStream(); // throws IoException - DhDataInputStream compressedInputStream = new DhDataInputStream(inputStream)) - { - renderSource = ColumnRenderLoader.INSTANCE.loadRenderSource(this, compressedInputStream, this.clientLevel); - } - catch (IOException ex) - { - throw new CompletionException(ex); - } - - return renderSource; - }, fileReaderThreads) - // TODO: Check for file version and only update if needed. - .thenCompose(renderSource -> this.updateRenderCacheAsync(renderSource)) - .whenComplete((renderSource, ex) -> - { - if (ex != null) - { - if (!LodUtil.isInterruptOrReject(ex)) - { - LOGGER.error("Error loading pos: "+this.pos+": ", ex); - } - - // set the render source to null to prevent instances where a corrupt or incomplete render source is returned - renderSource = null; - } - - this.renderSourceLoadFutureRef.set(null); - - this.cachedRenderDataSourceRef = new SoftReference<>(renderSource); - getSourceFuture.complete(renderSource); - }); - } - - return getSourceFuture; - } - // TODO merge with FullDataMetaFile - private InputStream getInputStream() throws IOException - { - MetaDataDto dto = this.renderDataSourceProvider.getRepo().getByPrimaryKey(this.pos.serialize()); - return new ByteArrayInputStream(dto.dataArray); - } - - - - //===============// - // cache handler // - //===============// - - public CompletableFuture updateRenderCacheAsync(ColumnRenderSource renderSource) - { - DebugRenderer.BoxWithLife debugBox = new DebugRenderer.BoxWithLife(new DebugRenderer.Box(renderSource.sectionPos, 74f, 86f, 0.1f, Color.red), 1.0, 32f, Color.green.darker()); - - - //// Skip updating the cache if the data file is already up-to-date - //FullDataMetaFile dataFile = this.fullDataSourceProvider.getDtoIfExist(this.pos); - //if (!ALWAYS_INVALIDATE_CACHE && dataFile != null && dataFile.baseMetaData != null && dataFile.baseMetaData.checksum == this.baseMetaData.dataVersion.get()) // TODO can we make it so the version comparisons either both use the checksum or the dataVersion? Comparing checksum and dataVersion is kinda confusing - //{ - // LOGGER.debug("Skipping render cache update for " + this.pos); - // renderSource.localVersion.incrementAndGet(); - // return CompletableFuture.completedFuture(renderSource); - //} - - - - // get the full data source - CompletableFuture fullDataSourceFuture = - this.fullDataSourceProvider.getAsync(renderSource.getSectionPos()) - .thenApply((fullDataSource) -> - { - debugBox.box.color = Color.yellow.darker(); - - //// get the metaFile's version - //FullDataMetaFile renderSourceMetaFile = this.fullDataSourceProvider.getDtoIfExist(this.pos); - //if (renderSourceMetaFile != null && renderSourceMetaFile.baseMetaData != null) - //{ - // renderDataVersionRef.value = renderSourceMetaFile.baseMetaData.checksum; - //} - - return fullDataSource; - }).exceptionally((ex) -> - { - LOGGER.error("Exception when getting data for updateCache()", ex); - return null; - }); - - - - // convert the full data source into a render source - CompletableFuture transformFuture = fullDataSourceFuture - .handle((fullDataSource, ex) -> - { - if (ex == null) - { - ColumnRenderSource newRenderSource = null; - try - { - newRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.clientLevel); - } - catch (Exception e) - { - LOGGER.error("Unable to transform full data to render data for pos: "+this.pos, e); - } - - try - { - if (newRenderSource != null) - { - renderSource.updateFromRenderSource(newRenderSource); - - // update the meta data - this.baseMetaData.dataVersion.set(Integer.MAX_VALUE); - this.baseMetaData.dataDetailLevel = renderSource.getDataDetailLevel(); - this.baseMetaData.dataType = RENDER_SOURCE_TYPE; - this.baseMetaData.binaryDataFormatVersion = renderSource.getRenderDataFormatVersion(); - - // save to file - this.save(renderSource); - } - } - catch (Throwable e) - { - LOGGER.error("Exception when writing render data for pos: "+this.pos, e); - } - } - else if (!LodUtil.isInterruptOrReject(ex)) - { - LOGGER.error("Exception when updating render file using data source: ", ex); - } - - debugBox.close(); - return renderSource; - }); - return transformFuture; - } - - - - //===============// - // file handling // - //===============// - - public CompletableFuture flushAndSaveAsync() - { - if (!this.renderDataSourceProvider.getRepo().existsWithPrimaryKey(this.pos.serialize())) - { - return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist. - } - - // FIXME: TODO: Change updateRenderSource to true. Currently is false cause a dead future making render handler hang, - // and that render cache aren't actually used really yet due to missing versioning atm. So disabling for now. - CompletableFuture getSourceFuture = this.getCachedDataSourceAsync(false); - if (getSourceFuture == null) - { - return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save. - } - - // Wait for the data to be read, which also flushes changes to the file. - return getSourceFuture.thenAccept((columnRenderSource) -> { /* discard the render source, it doesn't need to be returned */ }); - } - - /** writes the given {@link ColumnRenderSource} to file */ - private void save(ColumnRenderSource renderSource) - { - if (renderSource.isEmpty()) - { - // delete the empty data source - this.fullDataSourceProvider.getRepo().deleteByPrimaryKey(this.pos.serialize()); - this.doesDtoExist = false; - } - else - { - //LOGGER.info("Saving updated render file v[{}] at sect {}", this.metaData.dataVersion.get(), this.pos); - try - { - super.writeToDatabase((dhDataOutputStream) -> renderSource.writeData(dhDataOutputStream), this.renderDataSourceProvider.getRepo()); - this.doesDtoExist = true; - } - catch (IOException e) - { - LOGGER.error("Failed to save updated render data for pos "+this.pos, e); - } - } - } - - - - //=======// - // debug // - //=======// - - @Override - public void debugRender(DebugRenderer debugRenderer) - { - if (this.cachedRenderDataSourceRef.get() != null) - { - return; - //color = Color.GREEN; - } - - // determine the color - Color color = Color.black; - if (this.renderSourceLoadFutureRef.get() != null) - { - color = Color.BLUE; - } - else if (this.doesDtoExist) - { - color = Color.RED; - } - - debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 64, 72, 0.05f, color)); - } - - - - //================// - // helper methods // - //================// - - /** @return returns null if {@link RenderDataMetaFile#renderSourceLoadFutureRef} is empty and no cached {@link ColumnRenderSource} exists. */ - @Nullable - private CompletableFuture getCachedDataSourceAsync(boolean updateRenderSourceCache) - { - // check if another thread is already loading the data source - CompletableFuture renderSourceLoadFuture = this.renderSourceLoadFutureRef.get(); - if (renderSourceLoadFuture != null) - { - return renderSourceLoadFuture; - } - - - // attempt to get the cached render source - ColumnRenderSource cachedRenderDataSource = this.cachedRenderDataSourceRef.get(); - if (cachedRenderDataSource == null) - { - // no cached data exists and no one is trying to load it - return null; - } - else - { - // cached data exists - - if (!updateRenderSourceCache) - { - // just return the render source - return CompletableFuture.completedFuture(cachedRenderDataSource); - } - - // update the render cache, wait for the update to finish, then return the render source - - // Create a new future if one doesn't already exist - CompletableFuture newFuture = new CompletableFuture<>(); - CompletableFuture oldFuture = AtomicsUtil.compareAndExchange(this.renderSourceLoadFutureRef, null, newFuture); - - if (oldFuture != null) - { - // An update is already in progress, return its future. - return oldFuture; - } - else - { - this.updateRenderCacheAsync(cachedRenderDataSource) - // wait for the handler to finish before returning the renderSource - .handle((ignoredRenderSource, ex) -> - { - newFuture.complete(cachedRenderDataSource); - this.renderSourceLoadFutureRef.set(null); - - return null; - }); - return newFuture; - } - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 1e36d896f..5fd7b9705 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -20,7 +20,10 @@ package com.seibel.distanthorizons.core.file.renderfile; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandler; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; +import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; +import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -28,9 +31,8 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.sql.MetaDataDto; +import com.seibel.distanthorizons.core.sql.DataSourceDto; import com.seibel.distanthorizons.core.sql.RenderDataRepo; -import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPools; import org.apache.logging.log4j.Logger; @@ -39,28 +41,15 @@ import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; -public class RenderSourceFileHandler implements IRenderSourceProvider +public class RenderSourceFileHandler extends AbstractDataSourceHandler implements IRenderSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private final F3Screen.NestedMessage threadPoolMsg; - protected final ConcurrentHashMap loadedMetaFileBySectionPos = new ConcurrentHashMap<>(); - - private final IDhClientLevel clientLevel; - private final File saveDir; - /** This is the lowest (highest numeric) detail level that this {@link RenderSourceFileHandler} is keeping track of. */ - AtomicInteger topDetailLevelRef = new AtomicInteger(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); private final IFullDataSourceProvider fullDataSourceProvider; - private final WeakHashMap, ETaskType> taskTracker = new WeakHashMap<>(); - - public final RenderDataRepo renderDataRepo; - @Override - public RenderDataRepo getRepo() { return this.renderDataRepo; } - //=============// @@ -69,20 +58,18 @@ public class RenderSourceFileHandler implements IRenderSourceProvider public RenderSourceFileHandler(IFullDataSourceProvider sourceProvider, IDhClientLevel clientLevel, AbstractSaveStructure saveStructure) { + super(clientLevel, saveStructure, createRepo(clientLevel, saveStructure)); + this.fullDataSourceProvider = sourceProvider; - this.clientLevel = clientLevel; - this.saveDir = saveStructure.getRenderCacheFolder(clientLevel.getLevelWrapper()); - if (!this.saveDir.exists() && !this.saveDir.mkdirs()) - { - LOGGER.warn("Unable to create render data folder, file saving may fail."); - } - - this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log); + } + private static RenderDataRepo createRepo(IDhClientLevel clientLevel, AbstractSaveStructure saveStructure) + { + File saveDir = saveStructure.getRenderCacheFolder(clientLevel.getLevelWrapper()); try { - this.renderDataRepo = new RenderDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + return new RenderDataRepo("jdbc:sqlite", saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); } catch (SQLException e) { @@ -94,169 +81,36 @@ public class RenderSourceFileHandler implements IRenderSourceProvider - //===============// - // file handling // - //===============// + //====================// + // Abstract overrides // + //====================// - /** This call is thread safe and can be called concurrently from multiple threads. */ - @Override - public CompletableFuture readAsync(DhSectionPos pos) + @Override + protected ColumnRenderSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException + { return ColumnRenderSourceLoader.INSTANCE.loadRenderSource(dto, dto.getInputStream(), this.level); } + @Override + protected ColumnRenderSource createNewDataSourceFromExistingDtos(DhSectionPos pos) { - // don't continue if the handler has been shut down - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); - if (executor != null && executor.isTerminated()) + ColumnRenderSource renderDataSource; + + IFullDataSource fullDataSource = this.fullDataSourceProvider.get(pos); + if (fullDataSource != null) { - return CompletableFuture.completedFuture(null); + renderDataSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); } - - - - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - RenderDataMetaFile metaFile = this.getLoadOrMakeFile(pos); - if (metaFile == null) + else { - return CompletableFuture.completedFuture(ColumnRenderSource.createEmptyRenderSource(pos)); + renderDataSource = this.makeEmptyDataSource(pos); } - - CompletableFuture getDataSourceFuture = metaFile.getOrLoadCachedDataSourceAsync(executor) - .handle((renderSource, exception) -> - { - if (exception != null) - { - LOGGER.error("Uncaught error in readAsync for pos: " + pos + ". Error:", exception); - } - - return (renderSource != null) ? renderSource : ColumnRenderSource.createEmptyRenderSource(pos); - }); - - synchronized (this.taskTracker) - { - this.taskTracker.put(getDataSourceFuture, ETaskType.READ); - } - return getDataSourceFuture; - } - /** @return null if there was an issue */ - private RenderDataMetaFile getLoadOrMakeFile(DhSectionPos pos) - { - RenderDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos); - if (metaFile != null) - { - return metaFile; - } - - - MetaDataDto metaDataDto = this.renderDataRepo.getByPrimaryKey(pos.serialize()); - if (metaDataDto != null) - { - synchronized (this) - { - // A file exists, but isn't loaded yet. - - // Double check locking for loading file, as loading file means also loading the metadata, which - // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the - // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. - metaFile = this.loadedMetaFileBySectionPos.get(pos); - if (metaFile != null) - { - return metaFile; // someone else loaded it already. - } - - try - { - metaFile = RenderDataMetaFile.createFromExistingFile(this.fullDataSourceProvider, this, this.clientLevel, metaDataDto); - this.topDetailLevelRef.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.getDetailLevel())); - this.loadedMetaFileBySectionPos.put(pos, metaFile); - return metaFile; - } - catch (IOException e) - { - LOGGER.error("Failed to read meta data file at pos " + pos + ": ", e); - this.renderDataRepo.delete(metaDataDto); - } - } - } - - - // File does not exist, create it. - // In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS - // to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects. - try - { - metaFile = RenderDataMetaFile.createNewFileForPos(this.fullDataSourceProvider, this, this.clientLevel, pos); - } - catch (IOException e) - { - LOGGER.error("IOException on creating new render data file at "+pos, e); - return null; - } - - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - - // This is a Compare And Swap with expected null value. - RenderDataMetaFile metaFileCas = this.loadedMetaFileBySectionPos.putIfAbsent(pos, metaFile); - return (metaFileCas == null) ? metaFile : metaFileCas; + return renderDataSource; } + @Override + protected ColumnRenderSource makeEmptyDataSource(DhSectionPos pos) + { return ColumnRenderSource.createEmptyRenderSource(pos); } - //=============// - // data saving // - //=============// - - /** - * This call is thread safe and can be called concurrently from multiple threads.
- * This allows fast writes of new data to the render source, without having to wait for the data to be written to disk. - */ - @Override - public void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkDataView) - { - // convert to the lowest detail level so all detail levels are updated - this.writeChunkDataToFileRecursively(chunkDataView, DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - this.fullDataSourceProvider.updateDataSourcesWithChunkData(chunkDataView); - } - private void writeChunkDataToFileRecursively(ChunkSizedFullDataAccessor chunk, byte sectionDetailLevel) - { - DhSectionPos boundingPos = chunk.getSectionPos(); - DhSectionPos minSectionPos = boundingPos.convertNewToDetailLevel(sectionDetailLevel); - - DhSectionPos.DhMutableSectionPos fileSectionPos = new DhSectionPos.DhMutableSectionPos((byte)0, 0, 0); - - int width = (sectionDetailLevel > boundingPos.getDetailLevel()) ? 1 : boundingPos.getWidthCountForLowerDetailedSection(sectionDetailLevel); - for (int xOffset = 0; xOffset < width; xOffset++) - { - for (int zOffset = 0; zOffset < width; zOffset++) - { - fileSectionPos.mutate(sectionDetailLevel, minSectionPos.getX() + xOffset, minSectionPos.getZ() + zOffset); - RenderDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(fileSectionPos); // bypass the getLoadOrMakeFile() since we only want cached files. - if (metaFile != null) - { - metaFile.updateChunkIfSourceExistsAsync(chunk); - } - } - } - - if (sectionDetailLevel < this.topDetailLevelRef.get()) - { - this.writeChunkDataToFileRecursively(chunk, (byte) (sectionDetailLevel + 1)); - } - } - - - /** This call is thread safe and can be called concurrently from multiple threads. */ - @Override - public CompletableFuture flushAndSaveAsync() - { - ArrayList> futures = new ArrayList<>(); - for (RenderDataMetaFile metaFile : this.loadedMetaFileBySectionPos.values()) - { - futures.add(metaFile.flushAndSaveAsync()); - } - - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); - } - - //=========// // F3 menu // @@ -266,85 +120,35 @@ public class RenderSourceFileHandler implements IRenderSourceProvider private String[] f3Log() { ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); - String queueSize = executor != null ? executor.getQueue().size()+"" : "-"; - String completedTaskSize = executor != null ? executor.getCompletedTaskCount()+"" : "-"; - + String queueSize = (executor != null) ? executor.getQueue().size()+"" : "-"; + String completedTaskSize = (executor != null) ? executor.getCompletedTaskCount()+"" : "-"; ArrayList lines = new ArrayList<>(); - lines.add("Render Source File Handler [" + this.clientLevel.getClientLevelWrapper().getDimensionType().getDimensionName() + "]"); - lines.add(" Loaded files: " + this.loadedMetaFileBySectionPos.size()); + lines.add("File Handler [" + this.level.getLevelWrapper().getDimensionType().getDimensionName() + "]"); lines.add(" Thread pool tasks: " + queueSize + " (completed: " + completedTaskSize + ")"); + lines.add(" Unsaved render sources: " + this.unsavedDataSourceBySectionPos.size()); + lines.add(" Unsaved data sources: " + this.fullDataSourceProvider.getUnsavedDataSourceCount()); - int totalFutures = this.taskTracker.size(); - EnumMap tasksOutstanding = new EnumMap<>(ETaskType.class); - EnumMap tasksCompleted = new EnumMap<>(ETaskType.class); - for (ETaskType type : ETaskType.values()) - { - tasksOutstanding.put(type, 0); - tasksCompleted.put(type, 0); - } - - synchronized (this.taskTracker) - { - for (Map.Entry, ETaskType> entry : this.taskTracker.entrySet()) - { - if (entry.getKey().isDone()) - { - tasksCompleted.put(entry.getValue(), tasksCompleted.get(entry.getValue()) + 1); - } - else - { - tasksOutstanding.put(entry.getValue(), tasksOutstanding.get(entry.getValue()) + 1); - } - } - } - int totalOutstanding = tasksOutstanding.values().stream().mapToInt(Integer::intValue).sum(); - lines.add(" Futures: " + totalFutures + " (outstanding: " + totalOutstanding + ")"); - for (ETaskType type : ETaskType.values()) - { - lines.add(" " + type + ": " + tasksOutstanding.get(type) + " / " + (tasksOutstanding.get(type) + tasksCompleted.get(type))); - } return lines.toArray(new String[0]); } + public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView) + { + super.updateDataSourcesWithChunkData(chunkDataView); + this.fullDataSourceProvider.updateDataSourcesWithChunkData(chunkDataView); + } //=====================// - // clearing / shutdown // + // shutdown / clearing // //=====================// - @Override public void close() { - LOGGER.info("Closing " + this.getClass().getSimpleName() + " with [" + this.loadedMetaFileBySectionPos.size() + "] files..."); + super.close(); this.threadPoolMsg.close(); - this.renderDataRepo.close(); } - public void deleteRenderCache() - { - // clear the cached files - this.loadedMetaFileBySectionPos.clear(); - - // delete the render cache - this.renderDataRepo.deleteAll(); - } - - - - //================// - // helper classes // - //================// - - /** - * READ
- * UPDATE_READ_DATA
- * UPDATE
- * ON_LOADED
- */ - private enum ETaskType - { - READ, UPDATE_READ_DATA, UPDATE, ON_LOADED, - } + public void deleteRenderCache() { this.repo.deleteAll(); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 2fae17053..25a84c3f3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -30,7 +30,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.render.LodQuadTree; import com.seibel.distanthorizons.core.render.RenderBufferHandler; import com.seibel.distanthorizons.core.render.renderer.LodRenderer; @@ -42,7 +41,6 @@ import com.seibel.distanthorizons.coreapi.util.math.Mat4f; import org.apache.logging.log4j.Logger; import java.io.Closeable; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; public class ClientLevelModule implements Closeable @@ -178,14 +176,13 @@ public class ClientLevelModule implements Closeable //===============// // data handling // //===============// - public void writeChunkDataToFile(ChunkSizedFullDataAccessor data) + public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { - DhSectionPos pos = data.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); - ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { - ClientRenderState.renderSourceFileHandler.writeChunkDataToFile(pos, data); + ClientRenderState.renderSourceFileHandler.updateDataSourcesWithChunkData(data); + ClientRenderState.quadtree.reloadPos(data.sectionPos); } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 545cee3e1..504e7b1c1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -35,7 +35,6 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.io.File; -import java.util.concurrent.CompletableFuture; /** The level used when connected to a server */ public class DhClientLevel extends DhLevel implements IDhClientLevel @@ -116,7 +115,7 @@ public class DhClientLevel extends DhLevel implements IDhClientLevel public ILevelWrapper getLevelWrapper() { return levelWrapper; } @Override - public void saveWrites(ChunkSizedFullDataAccessor data) { this.clientside.writeChunkDataToFile(data); } + public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); } @Override public int getMinY() { return levelWrapper.getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index 913bbe80f..0d5416104 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -186,7 +186,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS public boolean hasSkyLight() { return this.serverLevelWrapper.hasSkyLight(); } @Override - public void saveWrites(ChunkSizedFullDataAccessor data) { this.clientside.writeChunkDataToFile(data); } + public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); } @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhLevel.java index b42266214..fe47c50dd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhLevel.java @@ -34,7 +34,7 @@ public abstract class DhLevel implements IDhLevel protected DhLevel() { this.chunkToLodBuilder = new ChunkToLodBuilder(); } - public abstract void saveWrites(ChunkSizedFullDataAccessor data); + public abstract void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data); @Override @@ -54,7 +54,7 @@ public abstract class DhLevel implements IDhLevel return; } - this.saveWrites(chunkSizedFullDataAccessor); + this.updateDataSourcesWithChunkData(chunkSizedFullDataAccessor); ApiEventInjector.INSTANCE.fireAllEvents( DhApiChunkModifiedEvent.class, new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunk.getChunkPos().x, chunk.getChunkPos().z)); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index a568d6a44..3be61540a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -50,7 +50,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel public void serverTick() { this.chunkToLodBuilder.tick(); } @Override - public void saveWrites(ChunkSizedFullDataAccessor data) + public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { DhSectionPos pos = data.getSectionPos(); pos = pos.convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhServerLevel.java index c34c19f7f..bfca9b22b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhServerLevel.java @@ -19,7 +19,6 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; public interface IDhServerLevel extends IDhWorldGenLevel diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 746b21602..2f9a9bda9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -417,10 +417,7 @@ public class LodQuadTree extends QuadTree implements AutoClose } } - // delete the cache files - // TODO this will only delete the files for this level/world this.renderSourceProvider.deleteRenderCache(); - LOGGER.info("Render cache invalidated, please wait a moment for everything to reload..."); } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index dbb6358cc..9472798cc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -178,7 +178,7 @@ public class LodRenderSection implements IDebugRenderable private void startLoadRenderSourceAsync() { - this.renderSourceLoadFuture = this.renderSourceProvider.readAsync(this.pos); + this.renderSourceLoadFuture = this.renderSourceProvider.getAsync(this.pos); this.renderSourceLoadFuture.whenComplete((renderSource, ex) -> { this.renderSource = renderSource; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java similarity index 72% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java index a79ee19cb..479da8595 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java @@ -20,19 +20,17 @@ package com.seibel.distanthorizons.core.sql; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.coreapi.util.StringUtil; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Map; -public abstract class AbstractMetaDataRepo extends AbstractDhRepo +public abstract class AbstractDataSourceRepo extends AbstractDhRepo { - public AbstractMetaDataRepo(String databaseType, String databaseLocation) throws SQLException + public AbstractDataSourceRepo(String databaseType, String databaseLocation) throws SQLException { - super(databaseType, databaseLocation, MetaDataDto.class); + super(databaseType, databaseLocation, DataSourceDto.class); } @@ -46,7 +44,7 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo //=======================// @Override - public MetaDataDto convertDictionaryToDto(Map objectMap) throws ClassCastException + public DataSourceDto convertDictionaryToDto(Map objectMap) throws ClassCastException { String posString = (String) objectMap.get("DhSectionPos"); DhSectionPos pos = DhSectionPos.deserialize(posString); @@ -61,22 +59,22 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo String dataType = (String) objectMap.get("DataType"); byte binaryDataFormatVersion = (Byte) objectMap.get("BinaryDataFormatVersion"); - BaseMetaData baseMetaData = new BaseMetaData(pos, - checksum, dataDetailLevel, worldGenStep, - dataType, binaryDataFormatVersion); - // binary data byte[] dataByteArray = (byte[]) objectMap.get("Data"); - MetaDataDto metaFile = new MetaDataDto(baseMetaData, dataByteArray); - return metaFile; + DataSourceDto dto = new DataSourceDto( + pos, + checksum, dataDetailLevel, worldGenStep, + dataType, binaryDataFormatVersion, + dataByteArray); + return dto; } @Override public String createSelectPrimaryKeySql(String primaryKey) { return "SELECT * FROM "+this.getTableName()+" WHERE DhSectionPos = '"+primaryKey+"'"; } @Override - public PreparedStatement createInsertStatement(MetaDataDto dto) throws SQLException + public PreparedStatement createInsertStatement(DataSourceDto dto) throws SQLException { String sql = "INSERT INTO "+this.getTableName() + "\n" + @@ -94,12 +92,12 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo int i = 1; statement.setObject(i++, dto.getPrimaryKeyString()); - statement.setObject(i++, dto.baseMetaData.checksum); - statement.setObject(i++, dto.baseMetaData.dataVersion); - statement.setObject(i++, dto.baseMetaData.dataDetailLevel); - statement.setObject(i++, dto.baseMetaData.worldGenStep); - statement.setObject(i++, dto.baseMetaData.dataType); - statement.setObject(i++, dto.baseMetaData.binaryDataFormatVersion); + statement.setObject(i++, dto.checksum); + statement.setObject(i++, dto.dataVersion); + statement.setObject(i++, dto.dataDetailLevel); + statement.setObject(i++, dto.worldGenStep); + statement.setObject(i++, dto.dataType); + statement.setObject(i++, dto.binaryDataFormatVersion); statement.setObject(i++, dto.dataArray); @@ -107,7 +105,7 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo } @Override - public PreparedStatement createUpdateStatement(MetaDataDto dto) throws SQLException + public PreparedStatement createUpdateStatement(DataSourceDto dto) throws SQLException { String sql = "UPDATE "+this.getTableName()+" \n" + @@ -126,12 +124,12 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo PreparedStatement statement = this.createPreparedStatement(sql); int i = 1; - statement.setObject(i++, dto.baseMetaData.checksum); - statement.setObject(i++, dto.baseMetaData.dataVersion); - statement.setObject(i++, dto.baseMetaData.dataDetailLevel); - statement.setObject(i++, dto.baseMetaData.worldGenStep); - statement.setObject(i++, dto.baseMetaData.dataType); - statement.setObject(i++, dto.baseMetaData.binaryDataFormatVersion); + statement.setObject(i++, dto.checksum); + statement.setObject(i++, dto.dataVersion); + statement.setObject(i++, dto.dataDetailLevel); + statement.setObject(i++, dto.worldGenStep); + statement.setObject(i++, dto.dataType); + statement.setObject(i++, dto.binaryDataFormatVersion); statement.setObject(i++, dto.dataArray); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java similarity index 64% rename from core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java index 465469508..8ad1ef558 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java @@ -17,21 +17,21 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.file.metaData; +package com.seibel.distanthorizons.core.sql; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; -/** - * Contains and represents the meta information ({@link DhSectionPos}, {@link BaseMetaData#dataDetailLevel}, etc.) - * stored at the beginning of files that use the {@link AbstractMetaDataContainerFile}.
- * Which, as of the time of writing, includes: {@link IFullDataSource} and {@link ColumnRenderSource} files. - */ -public class BaseMetaData +/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */ +public class DataSourceDto implements IBaseDTO { public DhSectionPos pos; public int checksum; @@ -46,9 +46,10 @@ public class BaseMetaData public String dataType; public byte binaryDataFormatVersion; + public final byte[] dataArray; - public BaseMetaData(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion) + public DataSourceDto(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) { this.pos = pos; this.checksum = checksum; @@ -57,6 +58,20 @@ public class BaseMetaData this.dataType = dataType; this.binaryDataFormatVersion = binaryDataFormatVersion; + + this.dataArray = dataArray; + } + + + @Override + public String getPrimaryKeyString() { return this.pos.serialize(); } + + /** @return a stream for the data contained in this DTO. */ + public DhDataInputStream getInputStream() throws IOException + { + InputStream inputStream = new ByteArrayInputStream(this.dataArray); + DhDataInputStream compressedStream = new DhDataInputStream(inputStream); + return compressedStream; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java index 53b25e8d4..7d6e3f633 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java @@ -19,12 +19,9 @@ package com.seibel.distanthorizons.core.sql; -import com.seibel.distanthorizons.core.pos.DhSectionPos; - -import java.sql.ResultSet; import java.sql.SQLException; -public class FullDataRepo extends AbstractMetaDataRepo +public class FullDataRepo extends AbstractDataSourceRepo { public static final String TABLE_NAME = "DhFullData"; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java deleted file mode 100644 index bdd1b5566..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.sql; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; -import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; - -/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */ -public class MetaDataDto implements IBaseDTO -{ - public final BaseMetaData baseMetaData; - public final byte[] dataArray; - - - public MetaDataDto(BaseMetaData baseMetaData, byte[] dataArray) - { - this.baseMetaData = baseMetaData; - this.dataArray = dataArray; - } - - - @Override - public String getPrimaryKeyString() { return this.baseMetaData.pos.serialize(); } - - /** @return a stream for the data contained in this DTO. */ - public DhDataInputStream getInputStream() throws IOException - { - InputStream inputStream = new ByteArrayInputStream(this.dataArray); - DhDataInputStream compressedStream = new DhDataInputStream(inputStream); - return compressedStream; - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java index 0375d27a3..8cbec7dfe 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java @@ -21,7 +21,7 @@ package com.seibel.distanthorizons.core.sql; import java.sql.SQLException; -public class RenderDataRepo extends AbstractMetaDataRepo +public class RenderDataRepo extends AbstractDataSourceRepo { public static final String TABLE_NAME = "DhRenderData";