From ec29ea8cc17c72ff416c7502aa3fbf42d4e1edde Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Jan 2024 14:02:45 -0600 Subject: [PATCH] Overhaul and simplify FullDataFileHandler --- .../distanthorizons/core/Initializer.java | 3 - .../methods/data/DhApiTerrainDataRepo.java | 2 +- .../core/api/internal/ServerApi.java | 12 - .../fullData/FullDataDownSampler.java | 4 +- .../loader/AbstractFullDataSourceLoader.java | 41 +- .../loader/CompleteFullDataSourceLoader.java | 2 +- ...hDetailIncompleteFullDataSourceLoader.java | 2 +- ...wDetailIncompleteFullDataSourceLoader.java | 2 +- .../sources/CompleteFullDataSource.java | 16 +- .../HighDetailIncompleteFullDataSource.java | 22 +- .../LowDetailIncompleteFullDataSource.java | 16 +- .../sources/interfaces/IFullDataSource.java | 25 +- .../interfaces/IStreamableFullDataSource.java | 26 +- .../core/file/DataSourceReferenceTracker.java | 233 ------ .../fullDatafile/FullDataFileHandler.java | 690 ++++++++------- .../file/fullDatafile/FullDataMetaFile.java | 787 ------------------ .../GeneratedFullDataFileHandler.java | 208 +---- .../fullDatafile/IFullDataSourceProvider.java | 44 +- .../AbstractMetaDataContainerFile.java | 1 + .../core/file/metaData/BaseMetaData.java | 5 +- .../file/renderfile/RenderDataMetaFile.java | 49 +- .../renderfile/RenderSourceFileHandler.java | 2 +- .../SubDimensionLevelMatcher.java | 2 +- .../generation/IWorldGenerationQueue.java | 1 - .../core/generation/WorldGenerationQueue.java | 6 - .../core/level/ClientLevelModule.java | 15 +- .../core/level/DhClientLevel.java | 6 - .../core/level/DhClientServerLevel.java | 28 +- .../core/level/DhServerLevel.java | 7 +- .../distanthorizons/core/level/IDhLevel.java | 1 - .../core/sql/AbstractMetaDataRepo.java | 32 +- .../distanthorizons/core/sql/MetaDataDto.java | 14 + .../core/world/DhClientServerWorld.java | 5 - .../core/world/DhClientWorld.java | 7 - .../core/world/DhServerWorld.java | 6 - .../distanthorizons/core/world/IDhWorld.java | 2 - 36 files changed, 549 insertions(+), 1775 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/DataSourceReferenceTracker.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index f5b8bc046..590f08bc7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.core; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.file.DataSourceReferenceTracker; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.world.DhApiWorldProxy; @@ -75,8 +74,6 @@ public class Initializer LOGGER.error("Programmer Error: No ["+IWrapperFactory.class.getSimpleName()+"] assigned to the DhApi."); } - DataSourceReferenceTracker.startGarbageCollectorBackgroundThread(); - } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 63b2f074a..e2d2c8d66 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -212,7 +212,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo try { // attempt to get/generate the data source for this section - IFullDataSource dataSource = level.getFileHandler().readAsync(sectionPos).get(); + IFullDataSource dataSource = level.getFileHandler().getAsync(sectionPos).get(); if (dataSource == null) { return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + sectionPos + "]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java index 9f48f183b..d85dd8639 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java @@ -139,18 +139,6 @@ public class ServerApi } } - @Deprecated // TODO not implemented, remove - public void serverSaveEvent() - { - LOGGER.debug("Server world " + SharedApi.getAbstractDhWorld() + " saving"); - - AbstractDhWorld serverWorld = SharedApi.getAbstractDhWorld(); - if (serverWorld != null) - { - serverWorld.saveAndFlush(); - } - } - //=======================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java index 3b333bf4a..ac8862668 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java @@ -56,7 +56,7 @@ public class FullDataDownSampler { for (int zOffset = 0; zOffset < sectionSizeNeeded; zOffset++) { - CompletableFuture future = provider.readAsync(new DhSectionPos( + CompletableFuture future = provider.getAsync(new DhSectionPos( CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset, basePos.z + zOffset)); future = future.whenComplete((source, ex) -> { if (ex == null && source != null && source instanceof CompleteFullDataSource) @@ -80,7 +80,7 @@ public class FullDataDownSampler { for (int zOffset = 0; zOffset < CompleteFullDataSource.WIDTH; zOffset++) { - CompletableFuture future = provider.readAsync(new DhSectionPos( + CompletableFuture future = provider.getAsync(new DhSectionPos( CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset * multiplier, basePos.z + zOffset * multiplier)); future = future.whenComplete((source, ex) -> { if (ex == null && source != null && source instanceof CompleteFullDataSource) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java index d8fda3392..a618a2773 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java @@ -21,10 +21,9 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.loader; import com.google.common.collect.HashMultimap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; +import com.seibel.distanthorizons.core.sql.MetaDataDto; import java.io.IOException; import java.util.*; @@ -120,31 +119,31 @@ public abstract class AbstractFullDataSourceLoader // data loading // //==============// + /** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */ + public IFullDataSource loadTemporaryDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException + { + IFullDataSource dataSource = this.tryGetPooledSource(); + if (dataSource != null) + { + dataSource.repopulateFromStream(dto, dto.getInputStream(), level); + } + else + { + dataSource = this.loadDataSource(dto, level); + } + + return dataSource; + } + /** * Can return null if any of the requirements aren't met. * * @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down */ - public IFullDataSource loadDataSource(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public IFullDataSource loadDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException { - IFullDataSource dataSource = this.createEmptyDataSource(dataFile.pos); - dataSource.populateFromStream(dataFile, inputStream, level); - return dataSource; - } - - /** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */ - public IFullDataSource loadTemporaryDataSource(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException - { - IFullDataSource dataSource = this.tryGetPooledSource(); - if (dataSource != null) - { - dataSource.repopulateFromStream(dataFile, inputStream, level); - } - else - { - dataSource = this.loadDataSource(dataFile, inputStream, level); - } - + IFullDataSource dataSource = this.createEmptyDataSource(dto.baseMetaData.pos); + dataSource.populateFromStream(dto, dto.getInputStream(), level); return dataSource; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java index 6964dd6c5..d9088d20d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java @@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFull public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader { - public CompleteFullDataSourceLoader() { super(CompleteFullDataSource.class, CompleteFullDataSource.DATA_SOURCE_TYPE, new byte[]{CompleteFullDataSource.DATA_FORMAT_VERSION}); } + public CompleteFullDataSourceLoader() { super(CompleteFullDataSource.class, CompleteFullDataSource.DATA_TYPE_NAME, new byte[]{CompleteFullDataSource.DATA_FORMAT_VERSION}); } @Override protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return CompleteFullDataSource.createEmpty(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java index aa7c246dd..9dcfb4f58 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java @@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIn public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader { - public HighDetailIncompleteFullDataSourceLoader() { super(HighDetailIncompleteFullDataSource.class, HighDetailIncompleteFullDataSource.DATA_SOURCE_TYPE, new byte[]{HighDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } + public HighDetailIncompleteFullDataSourceLoader() { super(HighDetailIncompleteFullDataSource.class, HighDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{HighDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } @Override protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return HighDetailIncompleteFullDataSource.createEmpty(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java index ac2491a92..49793c958 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java @@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailInc public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader { - public LowDetailIncompleteFullDataSourceLoader() { super(LowDetailIncompleteFullDataSource.class, LowDetailIncompleteFullDataSource.DATA_SOURCE_TYPE, new byte[]{LowDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } + public LowDetailIncompleteFullDataSourceLoader() { super(LowDetailIncompleteFullDataSource.class, LowDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{LowDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } @Override protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return LowDetailIncompleteFullDataSource.createEmpty(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index f13e4d6ae..939e12dfb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -25,12 +25,12 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArr import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.MetaDataDto; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -59,7 +59,9 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); public static final byte DATA_FORMAT_VERSION = 3; - public static final String DATA_SOURCE_TYPE = "CompleteFullDataSource"; + public static final String DATA_TYPE_NAME = "CompleteFullDataSource"; + @Override + public String getDataTypeName() { return DATA_TYPE_NAME; } private DhSectionPos sectionPos; private boolean isEmpty = true; @@ -103,12 +105,12 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetail = inputStream.readInt(); - if (dataDetail != dataFile.baseMetaData.dataDetailLevel) + if (dataDetail != dto.baseMetaData.dataDetailLevel) { - throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetail + " != " + dataFile.baseMetaData.dataDetailLevel)); + throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetail + " != " + dto.baseMetaData.dataDetailLevel)); } int width = inputStream.readInt(); @@ -185,7 +187,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return true; } @Override - public long[][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream dataInputStream) throws IOException + public long[][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream dataInputStream) throws IOException { // Data array length int dataPresentFlag = dataInputStream.readInt(); @@ -428,7 +430,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } @Override - public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; } + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } @Override public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java index a5264a1ed..bf48c8216 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java @@ -26,11 +26,11 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColum import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.MetaDataDto; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -72,7 +72,9 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo public static final byte MAX_SECTION_DETAIL = SECTION_SIZE_OFFSET + SPARSE_UNIT_DETAIL; public static final byte DATA_FORMAT_VERSION = 3; - public static final String DATA_SOURCE_TYPE = "HighDetailIncompleteFullDataSource"; + public static final String DATA_TYPE_NAME = "HighDetailIncompleteFullDataSource"; + @Override + public String getDataTypeName() { return DATA_TYPE_NAME; } protected final FullDataPointIdMap mapping; @@ -141,15 +143,15 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { - LodUtil.assertTrue(dataFile.pos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(dataFile.pos.getDetailLevel() <= MAX_SECTION_DETAIL); + LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() > SPARSE_UNIT_DETAIL); + LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() <= MAX_SECTION_DETAIL); int dataDetailLevel = inputStream.readShort(); - if (dataDetailLevel != dataFile.baseMetaData.dataDetailLevel) + if (dataDetailLevel != dto.baseMetaData.dataDetailLevel) { - throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dataFile.baseMetaData.dataDetailLevel+"]"); + throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.baseMetaData.dataDetailLevel+"]"); } // confirm that the detail level is correct @@ -254,11 +256,11 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo return true; } @Override - public long[][][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException + public long[][][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException { // calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize // TODO these values should be constant, should we still be calculating them like this? - int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.getDetailLevel() - SPARSE_UNIT_DETAIL); + int chunks = BitShiftUtil.powerOfTwo(dto.baseMetaData.pos.getDetailLevel() - SPARSE_UNIT_DETAIL); int dataPointsPerChunk = SECTION_SIZE / chunks; @@ -472,7 +474,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } @Override - public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; } + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } @Override public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java index 3145e2a1c..6422b0e52 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java @@ -26,11 +26,11 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColum import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.MetaDataDto; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -63,7 +63,9 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); public static final byte DATA_FORMAT_VERSION = 3; - public static final String DATA_SOURCE_TYPE = "LowDetailIncompleteFullDataSource"; + public static final String DATA_TYPE_NAME = "LowDetailIncompleteFullDataSource"; + @Override + public String getDataTypeName() { return DATA_TYPE_NAME; } private DhSectionPos sectionPos; @@ -118,12 +120,12 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp } @Override - public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetailLevel = inputStream.readInt(); - if (dataDetailLevel != dataFile.baseMetaData.dataDetailLevel) + if (dataDetailLevel != dto.baseMetaData.dataDetailLevel) { - throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dataFile.baseMetaData.dataDetailLevel)); + throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.baseMetaData.dataDetailLevel)); } int width = inputStream.readInt(); @@ -186,7 +188,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp return true; } @Override - public StreamDataPointContainer readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException + public StreamDataPointContainer readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException { // is source empty flag int dataPresentFlag = inputStream.readInt(); @@ -320,7 +322,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp @Override public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } @Override - public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; } + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } @Override public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java index cbd21b50a..ae098f723 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; @@ -27,15 +28,12 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedF import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.IFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; +import com.seibel.distanthorizons.core.sql.MetaDataDto; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import org.jetbrains.annotations.Nullable; import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedList; /** * Base for all Full Data Source objects.

@@ -61,9 +59,17 @@ public interface IFullDataSource DhSectionPos getSectionPos(); + /** + * Returns the name of this data source.
+ * Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse + * the binary data when read from file. + */ + String getDataTypeName(); + /** Returns the detail level of the data contained by this {@link IFullDataSource}. */ byte getDataDetailLevel(); - byte getBinaryDataFormatVersion(); + /** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */ + byte getDataFormatVersion(); EDhApiWorldGenerationStep getWorldGenStep(); void update(ChunkSizedFullDataAccessor data); @@ -101,7 +107,6 @@ public interface IFullDataSource // basic stream handling // //=======================// - // TODO make this blow up in IStreamableFullDataSource instead of the children /** * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. * @@ -112,15 +117,15 @@ public interface IFullDataSource /** * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. * - * @see IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel) + * @see IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) */ - void populateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; + void populateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; /** * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. * - * @see IStreamableFullDataSource#repopulateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel) + * @see IStreamableFullDataSource#repopulateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) */ - void repopulateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; + void repopulateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java index 9a12568e7..d3bb874e1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java @@ -22,8 +22,8 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.MetaDataDto; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; @@ -39,7 +39,7 @@ import java.io.IOException; * * @param defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}. * @param defines the object holding the data points, probably long[][] or long[][][]. - * {@link IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel) populateFromStream} + * {@link IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) populateFromStream} * for the full reasoning. */ public interface IStreamableFullDataSource extends IFullDataSource @@ -56,14 +56,14 @@ public interface IStreamableFullDataSource + * Confirms that the given {@link MetaDataDto} is valid for this {@link IStreamableFullDataSource}.
* This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed. * - * @throws IOException if the {@link FullDataMetaFile} isn't valid for this object. + * @throws IOException if the {@link MetaDataDto} isn't valid for this object. */ - SummaryDataType readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException; + SummaryDataType readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException; void setSourceSummaryData(SummaryDataType summaryData); /** @return true if any data points were present and written, false if this object was empty */ boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException; /** @return null if no data points were present */ - DataContainerType readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException; + DataContainerType readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException; void setDataPoints(DataContainerType dataPoints); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourceReferenceTracker.java b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourceReferenceTracker.java deleted file mode 100644 index 2e7f7d51a..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourceReferenceTracker.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; -import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; -import com.seibel.distanthorizons.core.file.renderfile.RenderDataMetaFile; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.util.ThreadUtil; -import org.apache.logging.log4j.Logger; - -import java.io.Closeable; -import java.lang.ref.ReferenceQueue; -import java.lang.ref.SoftReference; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * Keeps track of {@link FullDataMetaFile} and {@link RenderDataMetaFile}'s - * and handles freeing their underlying data sources if they go unused for a certain amount of time. - */ -public class DataSourceReferenceTracker -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final boolean LOG_GARBAGE_COLLECTIONS = false; - - /** How often the garbage collector thread will run */ - private static final long MS_BETWEEN_GARBAGE_CHECKS = TimeUnit.SECONDS.toMillis(60); - /** How long a data source has to go unused before it can be freed */ - private static final long MS_TO_EXPIRE_DATA_SOURCE = TimeUnit.SECONDS.toMillis(60); - - - // these queues are populated by the JVM's garbage collector after the assigned soft reference is freed - private static final ReferenceQueue FULL_DATA_GARBAGE_COLLECTED_QUEUE = new ReferenceQueue<>(); - private static final ReferenceQueue RENDER_DATA_GARBAGE_COLLECTED_QUEUE = new ReferenceQueue<>(); - - // TODO using a ConcurrentHashMap may or may not be the best choice here - private static final Set FULL_DATA_SOFT_REFS = ConcurrentHashMap.newKeySet(); - private static final Set RENDER_DATA_SOFT_REFS = ConcurrentHashMap.newKeySet(); - - private static final ThreadPoolExecutor GARBAGE_COLLECTOR_THREAD = ThreadUtil.makeSingleThreadPool("DataSourceReferenceTracker", ThreadUtil.MINIMUM_RELATIVE_PRIORITY); - - - - //=================// - // collector logic // - //=================// - - /** Warning: this should not be called more than once. */ - public static void startGarbageCollectorBackgroundThread() { /*GARBAGE_COLLECTOR_THREAD.execute(() -> garbageCollectorLoop());*/ } - private static void garbageCollectorLoop() - { - while(true) - { - try - { - runGarbageCollection(); - Thread.sleep(MS_BETWEEN_GARBAGE_CHECKS); - } - catch (InterruptedException e) - { - LOGGER.error("Garbage collector thread interrupted.", e); - } - catch (Exception e) - { - LOGGER.error("Unexpected data source garbage collector exception: " + e.getMessage(), e); - } - } - } - - public static void runGarbageCollection() - { - removeGarbageCollectedDataSources(); - removeExpiredDataSources(); - } - private static void removeGarbageCollectedDataSources() - { - FullDataSourceSoftRef garbageCollectedFullDataSoftRef = (FullDataSourceSoftRef) FULL_DATA_GARBAGE_COLLECTED_QUEUE.poll(); - while (garbageCollectedFullDataSoftRef != null) - { - if (LOG_GARBAGE_COLLECTIONS) - { - LOGGER.info("Full Data at pos: " + garbageCollectedFullDataSoftRef.metaFile.pos + " has been soft released."); - } - garbageCollectedFullDataSoftRef.close(); - - garbageCollectedFullDataSoftRef = (FullDataSourceSoftRef) FULL_DATA_GARBAGE_COLLECTED_QUEUE.poll(); - } - - RenderDataSourceSoftRef renderSoftRef = (RenderDataSourceSoftRef) RENDER_DATA_GARBAGE_COLLECTED_QUEUE.poll(); - while (renderSoftRef != null) - { - if (LOG_GARBAGE_COLLECTIONS) - { - LOGGER.info("Render Data at pos: " + renderSoftRef.metaFile.pos + " has been soft released."); - } - renderSoftRef.close(); - - renderSoftRef = (RenderDataSourceSoftRef) RENDER_DATA_GARBAGE_COLLECTED_QUEUE.poll(); - } - } - private static void removeExpiredDataSources() - { - // TODO merge these loops - FULL_DATA_SOFT_REFS.removeIf((fullDataSoftRef) -> - { - boolean remove = fullDataSoftRef.isDataSourceExpired() || (fullDataSoftRef.silentGet() == null); - if (remove) - { - fullDataSoftRef.clear(); - fullDataSoftRef.close(); - - if (LOG_GARBAGE_COLLECTIONS) - { - LOGGER.info("Full Data at pos: " + fullDataSoftRef.metaFile.pos + " has expired and will be released at the next GC. ["+FULL_DATA_SOFT_REFS.size()+"] Full data sources remain."); - } - } - - return remove; - }); - - // TODO merge these loops - RENDER_DATA_SOFT_REFS.removeIf((renderDataSoftRef) -> - { - boolean remove = renderDataSoftRef.isDataSourceExpired() || (renderDataSoftRef.silentGet() == null); - if (remove) - { - renderDataSoftRef.clear(); - renderDataSoftRef.close(); - - if (LOG_GARBAGE_COLLECTIONS) - { - LOGGER.info("Render Data at pos: " + renderDataSoftRef.metaFile.pos + " has expired and will be released at the next GC. ["+RENDER_DATA_SOFT_REFS.size()+"] Render data sources remain."); - } - } - - return remove; - }); - } - - - - //================// - // helper classes // - //================// - - public static class FullDataSourceSoftRef extends AbstractDataSourceSoftTracker - { - public FullDataSourceSoftRef(FullDataMetaFile metaFile, IFullDataSource data) - { - super(metaFile, data, FULL_DATA_GARBAGE_COLLECTED_QUEUE); - FULL_DATA_SOFT_REFS.add(this); - } - - @Override - public void close() { FULL_DATA_SOFT_REFS.remove(this); } - } - public static class RenderDataSourceSoftRef extends AbstractDataSourceSoftTracker - { - public RenderDataSourceSoftRef(RenderDataMetaFile metaFile, ColumnRenderSource data) - { - super(metaFile, data, RENDER_DATA_GARBAGE_COLLECTED_QUEUE); - RENDER_DATA_SOFT_REFS.add(this); - } - - @Override - public void close() { RENDER_DATA_SOFT_REFS.remove(this); } - } - - /** wrapper for a {@link SoftReference} so we can track and manually remove unused sources */ - public static abstract class AbstractDataSourceSoftTracker extends SoftReference implements Closeable - { - public final TMetaFile metaFile; - public final long createdMsTime; - - private long expirationMsTime; - - - - public AbstractDataSourceSoftTracker(TMetaFile metaFile, TDataSource dataSource, ReferenceQueue referenceQueue) - { - super(dataSource, referenceQueue); - this.metaFile = metaFile; - - this.createdMsTime = System.currentTimeMillis(); - this.expirationMsTime = System.currentTimeMillis(); - } - - - - public void updateLastAccessedTime() { this.expirationMsTime = System.currentTimeMillis() + MS_TO_EXPIRE_DATA_SOURCE; } - public long getExpirationMsTime() { return this.expirationMsTime; } - public boolean isDataSourceExpired() { return this.expirationMsTime > System.currentTimeMillis(); } - - - @Override - public TDataSource get() - { - this.updateLastAccessedTime(); - return super.get(); - } - - /** - * Gets the underlying datasource without updating the {@link AbstractDataSourceSoftTracker#expirationMsTime} - * Note: this still updates {@link SoftReference}'s timestamp variable which may prevent the JVM from - * marking this reference as valid for deletion. - */ - public TDataSource silentGet() { return super.get(); } - - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index f5c98ccbb..75347c8cd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -26,37 +26,55 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIn import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; +import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; +import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.sql.FullDataRepo; import com.seibel.distanthorizons.core.sql.MetaDataDto; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.distanthorizons.core.util.threading.ThreadPools; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.awt.*; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.ClosedChannelException; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; +import java.util.concurrent.locks.ReentrantLock; +import java.util.zip.Adler32; +import java.util.zip.CheckedOutputStream; public class FullDataFileHandler implements IFullDataSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final Timer DELAYED_SAVE_TIMER = new Timer(); + /** How long a data source must remain un-modified before being written to disk. */ + private static final int SAVE_DELAY_IN_MS = 4_000; - protected final ConcurrentHashMap loadedMetaFileBySectionPos = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap unsavedDataSourceBySectionPos = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap saveTimerTasksBySectionPos = new ConcurrentHashMap<>(); + protected final ReentrantLock[] updateLockArray; protected final IDhLevel level; protected final File saveDir; - protected final AtomicInteger topDetailLevelRef = new AtomicInteger(0); + + /** + * The highest numerical detail level known about. + * Used when determining which positions to update. + */ + protected final AtomicInteger topSectionDetailLevelRef; protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; public final FullDataRepo fullDataRepo; @@ -79,6 +97,14 @@ public class FullDataFileHandler implements IFullDataSourceProvider LOGGER.warn("Unable to create full data folder, file saving may fail."); } + // the lock array's length is double the number of CPU cores so the number of collisions + // should be relatively low without having too many extra locks + this.updateLockArray = new ReentrantLock[Runtime.getRuntime().availableProcessors() * 2]; + for (int i = 0; i < this.updateLockArray.length; i++) + { + this.updateLockArray[i] = new ReentrantLock(); + } + try { @@ -90,13 +116,17 @@ public class FullDataFileHandler implements IFullDataSourceProvider // or the database update failed throw new RuntimeException(e); } + + // determine the top detail level currently in the database + int maxSectionDetailLevel = this.fullDataRepo.getMaxSectionDetailLevel(); + this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel); } - //===============// - // file handling // - //===============// + //==============// + // data reading // + //==============// /** * Returns the {@link IFullDataSource} for the given section position.
@@ -107,251 +137,330 @@ public class FullDataFileHandler implements IFullDataSourceProvider * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ @Override - public CompletableFuture readAsync(DhSectionPos pos) + public CompletableFuture getAsync(DhSectionPos pos) { - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true); - if (metaFile == null) + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) { return CompletableFuture.completedFuture(null); } - - // future wrapper necessary in order to handle file read errors - CompletableFuture futureWrapper = new CompletableFuture<>(); - metaFile.getOrLoadCachedDataSourceAsync().exceptionally((e) -> - { - FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, e); - - futureWrapper.completeExceptionally(e); - return null; // return value doesn't matter - }) - .whenComplete((dataSource, e) -> - { - futureWrapper.complete(dataSource); - }); - - return futureWrapper; + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); } - - @Override - public FullDataMetaFile getFileIfExist(DhSectionPos pos) { return this.getLoadOrMakeFile(pos, false); } - protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile) + /** + * Should be used in internal methods where we are already running on a file handler thread. + * @see FullDataFileHandler#getAsync(DhSectionPos) + */ + protected IFullDataSource get(DhSectionPos pos) { - FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos); - if (metaFile != null) + // used the unsaved data source if present + if (this.unsavedDataSourceBySectionPos.containsKey(pos)) { - return metaFile; + return this.unsavedDataSourceBySectionPos.get(pos); } + // an unsaved data source isn't present + // check the database - // check if the file exists, but hasn't been loaded - MetaDataDto metaDataDto = this.fullDataRepo.getByPrimaryKey(pos.serialize()); - if (metaDataDto != null) - { - synchronized (this) - { - // Double check locking for loading file, as loading file means also loading the metadata, which - // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the - // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. - metaFile = this.loadedMetaFileBySectionPos.get(pos); - if (metaFile != null) - { - return metaFile; // someone else loaded it already. - } - - try - { - metaFile = FullDataMetaFile.createFromExistingDto(this, this.level, metaDataDto); - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - this.loadedMetaFileBySectionPos.put(pos, metaFile); - return metaFile; - } - catch (IOException e) - { - LOGGER.error("Failed to read meta data file at pos " + pos + ": ", e); - this.fullDataRepo.delete(metaDataDto); - } - } - } + // increase the top detail level if necessary + this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - if (!allowCreateFile) - { - return null; - } - - // File does not exist, create it. - // In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS - // to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects. + IFullDataSource dataSource = null; try { - metaFile = FullDataMetaFile.createNewDtoForPos(this, this.level, pos); - } - catch (IOException e) - { - LOGGER.error("IOException on creating new data file at {}", pos, e); - return null; - } - - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - - // This is a Compare And Swap with expected null value. - FullDataMetaFile metaFileCas = this.loadedMetaFileBySectionPos.putIfAbsent(pos, metaFile); - return metaFileCas == null ? metaFile : metaFileCas; - } - - /** - * Populates the preexistingFiles and missingFilePositions ArrayLists. - * - * @param preexistingFiles the list of {@link FullDataMetaFile}'s that have been created for the given position. - * @param missingFilePositions the list of {@link DhSectionPos}'s that don't have {@link FullDataMetaFile} created for them yet. - */ - protected void getDataFilesForPosition( - DhSectionPos effectivePos, DhSectionPos posAreaToGet, - ArrayList preexistingFiles, ArrayList missingFilePositions) - { - byte sectionDetail = posAreaToGet.getDetailLevel(); - boolean allEmpty = true; - - final DhSectionPos.DhMutableSectionPos subPos = new DhSectionPos.DhMutableSectionPos((byte)0, 0, 0); - - // get all existing files for this position - outerLoop: - while (--sectionDetail >= this.minDetailLevel) - { - DhLodPos minPos = posAreaToGet.getMinCornerLodPos().getCornerLodPos(sectionDetail); - int count = posAreaToGet.getSectionBBoxPos().getWidthAtDetail(sectionDetail); - - for (int xOffset = 0; xOffset < count; xOffset++) + MetaDataDto dto = this.fullDataRepo.getByPrimaryKey(pos.serialize()); + if (dto != null) { - for (int zOffset = 0; zOffset < count; zOffset++) - { - subPos.mutate(sectionDetail, xOffset + minPos.x, zOffset + minPos.z); - LodUtil.assertTrue(posAreaToGet.overlapsExactly(effectivePos) && subPos.overlapsExactly(posAreaToGet)); - - //TODO: The following check is temporary as we only sample corner points, which means - // on a very different level, we may not need the entire section at all. - if (!CompleteFullDataSource.firstDataPosCanAffectSecond(effectivePos, subPos)) - { - continue; - } - - // check if a file for this pos is loaded or exists - if (this.loadedMetaFileBySectionPos.containsKey(subPos) || this.fullDataRepo.existsWithPrimaryKey(subPos.serialize())) - { - allEmpty = false; - break outerLoop; - } - } - } - } - - if (allEmpty) - { - // there are no children to this quad tree, - // add this leaf's position - missingFilePositions.add(posAreaToGet); - } - else - { - // there are children in this quad tree, search them - this.recursiveGetDataFilesForPosition(0, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions); - this.recursiveGetDataFilesForPosition(1, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions); - this.recursiveGetDataFilesForPosition(2, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions); - this.recursiveGetDataFilesForPosition(3, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions); - } - } - private void recursiveGetDataFilesForPosition(int childIndex, DhSectionPos basePos, DhSectionPos pos, ArrayList preexistingFiles, ArrayList missingFilePositions) - { - DhSectionPos childPos = pos.getChildByIndex(childIndex); - if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos)) - { - // get or load the file if necessary - if (!this.loadedMetaFileBySectionPos.containsKey(childPos)) - { - this.getLoadOrMakeFile(childPos, false); - } - - - FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(childPos); - if (metaFile != null) - { - // we have reached a populated leaf node in the quad tree - preexistingFiles.add(metaFile); - } - else if (childPos.getDetailLevel() == this.minDetailLevel) - { - // we have reached an empty leaf node in the quad tree - missingFilePositions.add(childPos); + // load from file + AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.baseMetaData.dataType, dto.baseMetaData.binaryDataFormatVersion); + dataSource = loader.loadDataSource(dto, this.level); } else { - // recursively traverse down the tree - this.getDataFilesForPosition(basePos, childPos, preexistingFiles, missingFilePositions); + // attempt to create from any existing files + dataSource = this.createNewDataSourceFromExistingDtos(pos); } } - } - - public void ForEachFile(Consumer consumer) { this.loadedMetaFileBySectionPos.values().forEach(consumer); } - - - - //=============// - // data saving // - //=============// - - /** This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ - @Override - public void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkDataView) - { - DhSectionPos chunkSectionPos = chunkDataView.getSectionPos(); - LodUtil.assertTrue(chunkSectionPos.overlapsExactly(sectionPos), "Chunk " + chunkSectionPos + " does not overlap section " + sectionPos); - - chunkSectionPos = chunkSectionPos.convertNewToDetailLevel((byte) this.minDetailLevel); - this.writeChunkDataToMetaFile(chunkSectionPos, chunkDataView); - } - private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData) - { - FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos); - if (metaFile != null) + catch (InterruptedException ignore) { } + catch (IOException e) { - // there is a file for this position - metaFile.addToWriteQueue(chunkData); + LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); } - if (sectionPos.getDetailLevel() <= this.topDetailLevelRef.get()) + return dataSource; + } + /** Creates a new data source using any DTOs already present in the database. */ + protected IFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) + { + IIncompleteFullDataSource newFullDataSource = this.makeEmptyDataSource(pos); + + + boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); + if (showFullDataFileSampling) { - // recursively attempt to get the meta file for this position - this.writeChunkDataToMetaFile(sectionPos.getParentPos(), chunkData); + DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( + new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA), + 0.2, 32f)); } + + + // TODO replace with a SQL query, it should be much faster + ArrayList samplePosList = new ArrayList<>(); + ArrayList possibleChildList = new ArrayList<>(); + pos.forEachChild((childPos) -> + { + if (childPos.getDetailLevel() > this.minDetailLevel) + { + possibleChildList.add(childPos); + } + }); + while (possibleChildList.size() != 0) + { + DhSectionPos possiblePos = possibleChildList.remove(possibleChildList.size()-1); + if (this.fullDataRepo.existsWithPrimaryKey(possiblePos.serialize())) + { + samplePosList.add(possiblePos); + } + else + { + possiblePos.forEachChild((childPos) -> + { + if (childPos.getDetailLevel() > this.minDetailLevel) + { + possibleChildList.add(childPos); + } + }); + } + } + + + // read in the existing data + for (int i = 0; i < samplePosList.size(); i++) + { + DhSectionPos samplePos = samplePosList.get(i); + IFullDataSource sampleDataSource = this.get(samplePos); + if (sampleDataSource == null) + { + // no file was found, this is unexpected, but can be ignored + continue; + } + + if (showFullDataFileSampling) + { + DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( + new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA.darker()), + 0.2, 32f)); + } + + try + { + newFullDataSource.sampleFrom(sampleDataSource); + } + catch (Exception e) + { + LOGGER.warn("Unable to sample "+sampleDataSource.getSectionPos()+" into "+newFullDataSource.getSectionPos(), e); + } + } + + + // promotion may happen if all children are fully populated + return newFullDataSource.tryPromotingToCompleteDataSource(); } - /** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ - @Override - public CompletableFuture flushAndSaveAsync() - { - ArrayList> futures = new ArrayList<>(); - for (FullDataMetaFile metaFile : this.loadedMetaFileBySectionPos.values()) - { - futures.add(metaFile.flushAndSaveAsync()); - } - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); - } + + + //===============// + // data updating // + //===============// @Override - public CompletableFuture flushAndSaveAsync(DhSectionPos sectionPos) + public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView) { - FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos); - if (metaFile == null) + DhSectionPos chunkSectionPos = chunkDataView.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); + this.recursivelyUpdateDataSourcesAsync(chunkSectionPos, chunkDataView); + } + /** Updates every data source from this position up to {@link FullDataFileHandler#topSectionDetailLevelRef} */ + private void recursivelyUpdateDataSourcesAsync(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView) + { + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) { - return CompletableFuture.completedFuture(null); + return; + } + + // update up until we reach the highest available data source + if (pos.getDetailLevel() > this.topSectionDetailLevelRef.get()) + { + return; + } + + + executor.execute(() -> + { + DhSectionPos chunkSectionPos = chunkDataView.getSectionPos(); + LodUtil.assertTrue(chunkSectionPos.overlapsExactly(pos), "Update failed, chunk [" + chunkSectionPos + "] does not overlap section [" + pos + "]."); + + // update this pos + this.updateDataSourceAtPos(pos, chunkDataView); + + // recursively update the parent pos + DhSectionPos parentPos = pos.getParentPos(); + this.recursivelyUpdateDataSourcesAsync(parentPos, chunkDataView); + + }); + } + private void updateDataSourceAtPos(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView) + { + // a lock is necessary to prevent two threads from writing to the same position at once, + // if that happens only the second update will apply and the LOD will end up with hole(s) + ReentrantLock updateLock = this.getUpdateLockForPos(pos); + + try + { + updateLock.lock(); + + // get or create the data source + IFullDataSource fullDataSource = this.get(pos); + if (fullDataSource == null) + { + fullDataSource = this.makeEmptyDataSource(pos); + } + fullDataSource.update(chunkDataView); + + // try promoting the datasource + if (fullDataSource instanceof IIncompleteFullDataSource) + { + IIncompleteFullDataSource incompleteFullDataSource = (IIncompleteFullDataSource) fullDataSource; + fullDataSource = incompleteFullDataSource.tryPromotingToCompleteDataSource(); + } + + this.queueDelayedSave(fullDataSource); + } + catch (Exception e) + { + LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); + } + finally + { + updateLock.unlock(); + } + } + /** + * Queues the given data source to save after {@link FullDataFileHandler#SAVE_DELAY_IN_MS} + * milliseconds have passed without any additional modifications.

+ * + * This prevents repeatedly reading/writing the same data source to/from disk if said + * source is currently being updated via world gen or chunk modifications. + * This drastically reduces disk usage and improves performance. + */ + private void queueDelayedSave(IFullDataSource fullDataSource) + { + DhSectionPos pos = fullDataSource.getSectionPos(); + + // put the data source in memory until it can be flushed to disk + this.unsavedDataSourceBySectionPos.put(pos, fullDataSource); + + TimerTask task = new TimerTask() + { + @Override + public void run() + { + try + { + final IFullDataSource finalFullDataSource = FullDataFileHandler.this.unsavedDataSourceBySectionPos.remove(pos); + + // this can rarely happen due to imperfect concurrency handling, + // if the data source is null that just means it has already been saved so nothing needs to be done + if (finalFullDataSource != null) + { + FullDataFileHandler.this.writeDataSourceToFile(finalFullDataSource, (bufferedOutputStream) -> + { + try + { + finalFullDataSource.writeToStream(bufferedOutputStream, FullDataFileHandler.this.level); + } + catch (Exception e) + { + // if this try catch isn't included an empty exception will be thrown instead, which makes debugging extremely painful + LOGGER.error("Error writing data stream for pos: [" + finalFullDataSource.getSectionPos() + "], error: " + e.getMessage(), e); + } + }); + } + } + catch (ClosedByInterruptException e) // thrown by buffers that are interrupted + { + // expected if the file handler is shut down, the exception can be ignored + //LOGGER.warn("FullData file writing interrupted.", e); + } + catch (IOException e) + { + LOGGER.error("Failed to save updated data for section " + pos, e); + } + } + }; + DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS); + + // + TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task); + if (oldTask != null) + { + oldTask.cancel(); + } + } + private void writeDataSourceToFile(IFullDataSource fullDataSource, AbstractMetaDataContainerFile.IMetaDataWriterFunc dataWriterFunc) throws IOException + { + LodUtil.assertTrue(fullDataSource != null); + + + boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); + if (showFullDataFileStatus) + { + DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( + new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW), + 0.2, 16f)); + } + + try + { + // write the outputs to a stream to prep for writing to the database + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + + // the order of these streams is important, otherwise the checksum won't be calculated + CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); + // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, + // but since this stream will be closed immediately after writing anyway, it won't be an issue + DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut); + + dataWriterFunc.writeBinaryDataToStream(compressedOut); + + compressedOut.flush(); + int checksum = (int) checkedOut.getChecksum().getValue(); + byteArrayOutputStream.close(); + + + // save the DTO + BaseMetaData baseMetaData = new BaseMetaData(fullDataSource.getSectionPos(), checksum, + fullDataSource.getDataDetailLevel(), fullDataSource.getWorldGenStep(), fullDataSource.getDataTypeName(), + fullDataSource.getDataFormatVersion()); + MetaDataDto newDto = new MetaDataDto(baseMetaData, byteArrayOutputStream.toByteArray()); + this.fullDataRepo.save(newDto); + } + catch (ClosedChannelException e) // includes ClosedByInterruptException + { + // expected if the file handler is shut down, the exception can be ignored + //LOGGER.warn(AbstractMetaDataContainerFile.class.getSimpleName()+" file writing interrupted. Error: "+e.getMessage()); } - return metaFile.flushAndSaveAsync(); } + //================// + // helper methods // + //================// + + /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ + protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { @@ -360,136 +469,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider LowDetailIncompleteFullDataSource.createEmpty(pos); } - /** - * Populates the given data source using the given array of files - * @param usePooledDataSources if enabled the data sources necessary for this sampling will not be stored beyond what is necessary for the sampling. - * This helps reduce garbage collector pressure if the data sources will never be used again. - */ - protected CompletableFuture sampleFromFileArray(IIncompleteFullDataSource recipientFullDataSource, ArrayList existingFiles, boolean usePooledDataSources) - { - boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileSampling.get(); - if (showFullDataFileSampling) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(recipientFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA), - 0.2, 32f)); - } - - // read in the existing data - final ArrayList> sampleDataFutures = new ArrayList<>(existingFiles.size()); - for (int i = 0; i < existingFiles.size(); i++) - { - FullDataMetaFile existingFile = existingFiles.get(i); - - - CompletableFuture loadFileFuture = usePooledDataSources ? existingFile.getDataSourceWithoutCachingAsync() : existingFile.getOrLoadCachedDataSourceAsync(); - - CompletableFuture sampleSourceFuture = loadFileFuture.whenComplete((existingFullDataSource, ex) -> - { - if (existingFullDataSource == null || ex != null) - { - // Ignore file read errors - //LOGGER.warn(recipientFullDataSource.getSectionPos()+" sample from, file read error for file "+existingFile.pos+": "+ex.getMessage(), ex); - return; - } - - // can happen if data source caching isn't working correctly - LodUtil.assertTrue(existingFile.pos.equals(existingFullDataSource.getSectionPos()), "Data source returned the wrong position, pooled data source: ["+usePooledDataSources+"]. Expected: ["+existingFile.pos+"] actual: ["+existingFullDataSource.getSectionPos()+"]."); - - - if (showFullDataFileSampling) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(recipientFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA.darker()), - 0.2, 32f)); - } - - try - { - recipientFullDataSource.sampleFrom(existingFullDataSource); - } - catch (Exception e) - { - LOGGER.warn("Unable to sample "+existingFullDataSource.getSectionPos()+" into "+recipientFullDataSource.getSectionPos(), e); - //throw e; - } - - - // return the pooled data source if necessary - if (usePooledDataSources) - { - // pooling temporary data sources massively reduces garbage collector overhead when just sampling (going from ~8 GB/sec to ~90 MB/sec) - - // get the data loader - AbstractFullDataSourceLoader dataSourceLoader; - if (existingFile.fullDataSourceLoader != null) - { - dataSourceLoader = existingFile.fullDataSourceLoader; - } - else - { - // shouldn't normally happen, but sometimes does - dataSourceLoader = AbstractFullDataSourceLoader.getLoader(existingFile.baseMetaData.dataType, existingFile.baseMetaData.binaryDataFormatVersion); - } - - dataSourceLoader.returnPooledDataSource(existingFullDataSource); - } - }); - - sampleDataFutures.add(sampleSourceFuture); - } - return CompletableFuture.allOf(sampleDataFutures.toArray(new CompletableFuture[0])) - .thenApply(voidObj -> recipientFullDataSource); - } - - protected void makeFiles(ArrayList posList, ArrayList output) - { - for (DhSectionPos missingPos : posList) - { - FullDataMetaFile newFile = this.getLoadOrMakeFile(missingPos, true); - if (newFile != null) - { - output.add(newFile); - } - } - } - - @Override - public CompletableFuture onDataFileCreatedAsync(FullDataMetaFile file) - { - DhSectionPos pos = file.pos; - IIncompleteFullDataSource source = this.makeEmptyDataSource(pos); - ArrayList existFiles = new ArrayList<>(); - ArrayList missing = new ArrayList<>(); - this.getDataFilesForPosition(pos, pos, existFiles, missing); - LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty()); - if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) - { - // None exist. - return CompletableFuture.completedFuture(source); - } - else - { - this.makeFiles(missing, existFiles); - return this.sampleFromFileArray(source, existFiles, true).thenApply(IIncompleteFullDataSource::tryPromotingToCompleteDataSource) - .exceptionally((e) -> - { - FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, e); - return null; - }); - } - } - protected FullDataMetaFile removeCorruptedFile(DhSectionPos pos, Throwable exception) - { - LOGGER.error("Error reading Data file [" + pos + "]", exception); - - this.fullDataRepo.deleteByPrimaryKey(pos.serialize()); - // remove the FullDataMetaFile since the old one was corrupted - this.loadedMetaFileBySectionPos.remove(pos); - // create a new FullDataMetaFile to write new data to - return this.getLoadOrMakeFile(pos, true); - } - //=========// @@ -497,9 +476,24 @@ public class FullDataFileHandler implements IFullDataSourceProvider //=========// @Override - public void close() { - FullDataMetaFile.checkAndLogPhantomDataSourceLifeCycles(); - this.fullDataRepo.close(); + public void close() + { + LOGGER.info("Closing file handler for level: ["+this.level+"], saving ["+this.saveTimerTasksBySectionPos.size()+"] positions."); + + Enumeration list = this.saveTimerTasksBySectionPos.keys(); + while (list.hasMoreElements()) + { + DhSectionPos pos = list.nextElement(); + TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos); + if (saveTask != null) + { + saveTask.run(); + saveTask.cancel(); + } + } + + LOGGER.info("File handler saving complete, closing repo."); + this.fullDataRepo.close(); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java deleted file mode 100644 index 52f7ae1f9..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java +++ /dev/null @@ -1,787 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.fullDatafile; - -import java.awt.*; -import java.io.*; -import java.lang.ref.*; -import java.nio.channels.ClosedByInterruptException; -import java.util.Set; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.file.DataSourceReferenceTracker; -import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; -import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; -import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.sql.MetaDataDto; -import com.seibel.distanthorizons.core.util.AtomicsUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; -import org.apache.logging.log4j.Logger; - -/** Represents a File that contains a {@link IFullDataSource}. */ -public class FullDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName()); - - // === Object lifetime tracking === - /** if true both data source creation and garbage collection will be logged */ - private static final boolean LOG_DATA_SOURCE_LIVES = false; - private static final ReferenceQueue LIFE_CYCLE_DEBUG_QUEUE = new ReferenceQueue<>(); - private static final ReferenceQueue SOFT_REF_DEBUG_QUEUE = new ReferenceQueue<>(); - private static final Set LIFE_CYCLE_DEBUG_SET = ConcurrentHashMap.newKeySet(); - private static final Set SOFT_REF_DEBUG_SET = ConcurrentHashMap.newKeySet(); - // =========================== - - - - public boolean doesDtoExist; - /** indicates if this file has been checked for missing sections to generate or not */ - public boolean genQueueChecked = false; - - public AbstractFullDataSourceLoader fullDataSourceLoader; - public Class fullDataSourceClass; - - - private volatile boolean needsUpdate = false; - - private final IDhLevel level; - private final IFullDataSourceProvider fullDataSourceProvider; - - /** - * Can be cleared if the garbage collector determines there isn't enough space.

- * - * When clearing, don't set to null, instead create a SoftReference containing null. - * This makes null checks simpler. - */ - private DataSourceReferenceTracker.FullDataSourceSoftRef cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this,null); - // two different load futures are used to - // prevent accidentally returning a pooled (non-cached) data source - private final AtomicReference> cachedDataSourceLoadFutureRef = new AtomicReference<>(null); - private final AtomicReference> pooledDataSourceLoadFutureRef = new AtomicReference<>(null); - - - // === Concurrent Write tracking === - private final AtomicReference writeQueueRef = new AtomicReference<>(new GuardedMultiAppendQueue()); - private GuardedMultiAppendQueue backWriteQueue = new GuardedMultiAppendQueue(); - // =========================== - - - - //==============// - // constructors // - //==============// - - /** - * NOTE: should only be used if there is NOT an existing file. - * @throws IOException if a file already exists for this position - */ - public static FullDataMetaFile createNewDtoForPos(IFullDataSourceProvider fullDataSourceProvider, IDhLevel clientLevel, DhSectionPos pos) throws IOException { return new FullDataMetaFile(fullDataSourceProvider, clientLevel, pos); } - private FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, DhSectionPos pos) throws IOException - { - super(pos); - checkAndLogPhantomDataSourceLifeCycles(); - - this.fullDataSourceProvider = fullDataSourceProvider; - this.level = level; - LodUtil.assertTrue(this.baseMetaData == null); - this.doesDtoExist = false; - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus); - } - - - /** - * NOTE: should only be used if there IS an existing file. - * @throws IOException if the file was formatted incorrectly - * @throws FileNotFoundException if no file exists for the given path - */ - public static FullDataMetaFile createFromExistingDto(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, MetaDataDto metaDataDto) throws IOException { return new FullDataMetaFile(fullDataSourceProvider, level, metaDataDto); } - private FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, MetaDataDto metaDataDto) throws IOException - { - super(metaDataDto.baseMetaData); - checkAndLogPhantomDataSourceLifeCycles(); - - this.fullDataSourceProvider = fullDataSourceProvider; - this.level = level; - LodUtil.assertTrue(this.baseMetaData != null); - this.doesDtoExist = true; - - this.fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(this.baseMetaData.dataType, this.baseMetaData.binaryDataFormatVersion); - if (this.fullDataSourceLoader == null) - { - throw new IOException("Invalid file: Data type loader not found: " + this.baseMetaData.dataType + "(v" + this.baseMetaData.binaryDataFormatVersion + ")"); - } - - this.fullDataSourceClass = this.fullDataSourceLoader.fullDataSourceClass; - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus); - } - - - - //==========// - // get data // - //==========// - - /** - * Try get cached data source. Used for temp impl of re-queueing world gen tasks. - * (Read-only access! As writes should always be done async) - */ - public IFullDataSource getCachedDataSourceNowOrNull() - { - checkAndLogPhantomDataSourceLifeCycles(); - return this.cachedFullDataSourceRef.get(); - } - - /** @return if any data was cleared */ - public boolean clearCachedDataSource() - { - boolean dataExists = this.cachedFullDataSourceRef.get() != null; - if (dataExists) - { - this.cachedFullDataSourceRef.close(); - this.cachedFullDataSourceRef.clear(); - } - - return dataExists; - } - - - - public CompletableFuture getDataSourceWithoutCachingAsync() { return this.getOrLoadCachedDataSourceAsync(true); } - public CompletableFuture getOrLoadCachedDataSourceAsync() { return this.getOrLoadCachedDataSourceAsync(true); } - /** - * Synchronized to help prevent issues where multiple threads try to read as cached and un-cached at the same time. - * Hopefully isn't necessary and could potentially be removed in the future. - */ - private synchronized CompletableFuture getOrLoadCachedDataSourceAsync(boolean cacheLoadingSource) - { - checkAndLogPhantomDataSourceLifeCycles(); - - AtomicReference> dataSourceLoadFutureRef = cacheLoadingSource ? this.cachedDataSourceLoadFutureRef : this.pooledDataSourceLoadFutureRef; - - - - //========================// - // use the pre-existing // - // load future if present // - //========================// - - CompletableFuture preExistingLoadFuture = dataSourceLoadFutureRef.get(); - if (preExistingLoadFuture != null) - { - return preExistingLoadFuture; - } - - - - //========================// - // attempt to get the // - // cached data if present // - //========================// - - CompletableFuture potentialLoadFuture = null; - if (cacheLoadingSource) - { - potentialLoadFuture = this.getCachedDataSourceAndUpdateIfNeededAsync(); - } - - if (potentialLoadFuture != null) - { - // return the in-process future - return potentialLoadFuture; - } - else - { - // there is no cached data, we'll have to load it - - // create a new load future if necessary - potentialLoadFuture = new CompletableFuture<>(); - if (!dataSourceLoadFutureRef.compareAndSet(null, potentialLoadFuture)) - { - // two threads attempted to start this job at the same time, only use the first future - // (shouldn't happen since this method is synchronized, but just in case) - potentialLoadFuture = dataSourceLoadFutureRef.get(); - } - } - - - - final CompletableFuture dataSourceLoadFuture = potentialLoadFuture; - if (!this.doesDtoExist) - { - //==================// - // create a new DTO // - // and data source // - //==================// - - this.fullDataSourceProvider.onDataFileCreatedAsync(this) - .thenApply((fullDataSource) -> - { - AbstractFullDataSourceLoader dataSourceLoader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getBinaryDataFormatVersion()); - - this.baseMetaData = new BaseMetaData( - fullDataSource.getSectionPos(), -1, - fullDataSource.getDataDetailLevel(), fullDataSource.getWorldGenStep(), - (dataSourceLoader == null ? null : dataSourceLoader.datatype), fullDataSource.getBinaryDataFormatVersion(), Long.MAX_VALUE); - - return fullDataSource; - }) - .thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource)) - .thenAccept((fullDataSource) -> - { - dataSourceLoadFuture.complete(fullDataSource); - dataSourceLoadFutureRef.set(null); - }); - } - else - { - //=========================// - // load the data from file // - //=========================// - - if (this.baseMetaData == null) - { - throw new IllegalStateException("Meta data not loaded!"); - } - - - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); - if (executor != null && !executor.isTerminated()) - { - // load the data source - - CompletableFuture.supplyAsync(() -> - { - // Load the file. - IFullDataSource fullDataSource; - try (InputStream inputStream = this.getInputStream(); - DhDataInputStream compressedStream = new DhDataInputStream(inputStream)) - { - if (cacheLoadingSource) - { - fullDataSource = this.fullDataSourceLoader.loadDataSource(this, compressedStream, this.level); - } - else - { - fullDataSource = this.fullDataSourceLoader.loadTemporaryDataSource(this, compressedStream, this.level); - } - } - catch (Exception ex) - { - LOGGER.error("Full Data Load error for pos ["+this.pos+"], error: "+ ex.getMessage(), ex); - - dataSourceLoadFuture.completeExceptionally(ex); - dataSourceLoadFutureRef.set(null); - - // can happen if there is a missing file or the file was incorrectly formatted, or terminated early - throw new CompletionException(ex); - } - return fullDataSource; - }, executor) - .thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource)) - .thenAccept((fullDataSource) -> - { - dataSourceLoadFuture.complete(fullDataSource); - dataSourceLoadFutureRef.set(null); - }); - } - else - { - // don't load anything if the provider has been shut down - dataSourceLoadFuture.complete(null); - dataSourceLoadFutureRef.set(null); - return dataSourceLoadFuture; - } - } - - return dataSourceLoadFuture; - } - /** @return returns null if {@link FullDataMetaFile#cachedFullDataSourceRef} is empty and no cached {@link IFullDataSource} exists. */ - private CompletableFuture getCachedDataSourceAndUpdateIfNeededAsync() - { - // attempt to get the cached data source - IFullDataSource cachedFullDataSource = this.cachedFullDataSourceRef.get(); - if (cachedFullDataSource == null) - { - // no cached data exists and no one is trying to load it - return null; - } - else - { - // cached data exists - - boolean dataNeedsUpdating = !this.writeQueueRef.get().queue.isEmpty() || this.needsUpdate; - if (!dataNeedsUpdating) - { - // return the cached data - return CompletableFuture.completedFuture(cachedFullDataSource); - } - else - { - // update the data using the write queue, wait for the update to finish, then return the data source - - // Create a new future if one doesn't already exist - CompletableFuture newFuture = new CompletableFuture<>(); - CompletableFuture oldFuture = AtomicsUtil.compareAndExchange(this.cachedDataSourceLoadFutureRef, null, newFuture); - - if (oldFuture != null) - { - // An update is already in progress, return its future. - return oldFuture; - } - else - { - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); - if (executor != null && !executor.isTerminated()) - { - // wait for the update to finish before returning the data source - - CompletableFuture.supplyAsync(() -> cachedFullDataSource, executor) - .thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, true)) - .thenAccept((fullDataSource) -> - { - newFuture.complete(fullDataSource); - this.cachedDataSourceLoadFutureRef.set(null); - }); - } - else - { - // don't update anything if the provider has been shut down - this.cachedDataSourceLoadFutureRef.set(null); - newFuture.complete(null); - } - - return newFuture; - } - } - } - } - - - - //===============// - // data updating // - //===============// - - /** - * Adds the given {@link ChunkSizedFullDataAccessor} to the write queue, - * which will be applied to the object at some undefined time in the future. - */ - public void addToWriteQueue(ChunkSizedFullDataAccessor chunkAccessor) - { - checkAndLogPhantomDataSourceLifeCycles(); - - DhLodPos chunkLodPos = new DhLodPos(LodUtil.CHUNK_DETAIL_LEVEL, chunkAccessor.chunkPos.x, chunkAccessor.chunkPos.z); - - LodUtil.assertTrue(this.pos.getSectionBBoxPos().overlapsExactly(chunkLodPos), "Chunk pos " + chunkLodPos + " doesn't exactly overlap with section " + this.pos); - //LOGGER.info("Write Chunk {} to file {}", chunkPos, pos); - - GuardedMultiAppendQueue writeQueue = this.writeQueueRef.get(); - // Using read lock is OK, because the queue's underlying data structure is thread-safe. - // This lock is only used to insure on polling the queue, that the queue is not being - // modified by another thread. - ReentrantReadWriteLock.ReadLock appendLock = writeQueue.appendLock.readLock(); - appendLock.lock(); - try - { - writeQueue.queue.add(chunkAccessor); - } - finally - { - appendLock.unlock(); - } - - this.flushAndSaveAsync(); - //LOGGER.info("write queue length for pos "+this.pos+": " + writeQueue.queue.size()); - } - - - /** Applies any queued {@link ChunkSizedFullDataAccessor} to this metadata's {@link IFullDataSource} and writes the data to file. */ - public CompletableFuture flushAndSaveAsync() - { - checkAndLogPhantomDataSourceLifeCycles(); - boolean isEmpty = this.writeQueueRef.get().queue.isEmpty() && !this.needsUpdate; - if (!isEmpty) - { - // This will flush the data to disk. - return this.getDataSourceWithoutCachingAsync().thenApply((fullDataSource) -> null /* ignore the result, just wait for the load to finish*/ ); - } - else - { - return CompletableFuture.completedFuture(null); - } - } - - - - - - - public void markNeedsUpdate() { this.needsUpdate = true; } - - - - //===========// - // debugging // - //===========// - - /** can be used to log when data sources have been garbage collected */ - public static void checkAndLogPhantomDataSourceLifeCycles() - { - DataObjTracker phantomRef = (DataObjTracker) LIFE_CYCLE_DEBUG_QUEUE.poll(); - // wait for the tracker to be garbage collected - while (phantomRef != null) - { - if (LOG_DATA_SOURCE_LIVES) - { - LOGGER.info("Full Data at pos: " + phantomRef.pos + " has been freed. [" + LIFE_CYCLE_DEBUG_SET.size() + "] Full Data sources remaining."); - } - - phantomRef.close(); - phantomRef = (DataObjTracker) LIFE_CYCLE_DEBUG_QUEUE.poll(); - } - - - DataObjSoftTracker softRef = (DataObjSoftTracker) SOFT_REF_DEBUG_QUEUE.poll(); - while (softRef != null) - { - if (LOG_DATA_SOURCE_LIVES) - { - LOGGER.info("Full Data at pos: " + softRef.file.pos + " has been soft released."); - } - - softRef.close(); - softRef = (DataObjSoftTracker) SOFT_REF_DEBUG_QUEUE.poll(); - } - } - - @Override - public void debugRender(DebugRenderer debugRenderer) - { - if (this.pos.getDetailLevel() > DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL) - { - return; - } - - - - if (this.needsUpdate) - { - debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 80f, 96f, 0.05f, Color.red)); - } - - - IFullDataSource cachedDataSource = this.cachedFullDataSourceRef.get(); - boolean needsUpdate = !this.writeQueueRef.get().queue.isEmpty() || this.needsUpdate; - - // determine the color - Color color = Color.black; - if (cachedDataSource != null) - { - if (cachedDataSource instanceof CompleteFullDataSource) - { - color = Color.GREEN; - } - else - { - color = Color.YELLOW; - } - } - else if (this.cachedDataSourceLoadFutureRef.get() != null) - { - color = Color.BLUE; - } - else if (this.doesDtoExist) - { - color = Color.RED; - } - else if (needsUpdate) - { - color = color.darker().darker(); - } - - debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 80f, 96f, 0.05f, color)); - } - - - - //================// - // helper methods // - //================// - - // TODO merge with RenderDataMetaFile - /** @return a stream for the data contained in this file, skips the metadata from {@link AbstractMetaDataContainerFile}. */ - private InputStream getInputStream() throws IOException - { - MetaDataDto dto = this.fullDataSourceProvider.getRepo().getByPrimaryKey(this.pos.serialize()); - return new ByteArrayInputStream(dto.dataArray); - } - - /** - * Applies the {@link FullDataMetaFile#writeQueueRef} to the current {@link IFullDataSource} - * and stores the result in {@link FullDataMetaFile#cachedFullDataSourceRef}. - */ - @SuppressWarnings("resource") // due to DataObjTracker and DataObjSoftTracker being created outside a try-catch block - private CompletableFuture applyWriteQueueAndSaveAsync(IFullDataSource fullDataSourceToUpdate, boolean cacheLoadingSource) - { - CompletableFuture completionFuture = new CompletableFuture<>(); - - - boolean dataChanged = this.applyWriteQueueToFullDataSource(fullDataSourceToUpdate); - this.needsUpdate = false; - - // attempt to promote the data source - if (fullDataSourceToUpdate instanceof IIncompleteFullDataSource) - { - IFullDataSource newSource = ((IIncompleteFullDataSource) fullDataSourceToUpdate).tryPromotingToCompleteDataSource(); - dataChanged |= (newSource != fullDataSourceToUpdate); - fullDataSourceToUpdate = newSource; - } - - // the provider may need to modify other files based on this data source changing - this.fullDataSourceProvider.onDataFileUpdateAsync(fullDataSourceToUpdate, this, dataChanged) - .whenComplete((dataFileUpdateResult, ex) -> - { - if (ex != null && !LodUtil.isInterruptOrReject(ex)) - { - LOGGER.error("Error updating full meta file ["+this.pos+"]: ", ex); - } - - IFullDataSource fullDataSource = dataFileUpdateResult.fullDataSource; - boolean dataSourceChanged = dataFileUpdateResult.dataSourceChanged; - - - // only save to file if something was changed - if (dataSourceChanged) - { - this.writeDataSource(fullDataSource); - } - - // keep track of non-null data sources - if (fullDataSource != null) - { - new DataObjTracker(fullDataSource); - new DataObjSoftTracker(this, fullDataSource); - } - - - boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); - boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileSampling.get(); - if (showFullDataFileStatus || showFullDataFileSampling) - { - Color color = dataSourceChanged ? Color.GREEN : Color.GREEN.darker().darker(); - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(this.pos, 64f, 72f, 0.03f, color), - 0.2, 32f)); - } - - - if (cacheLoadingSource) - { - if (fullDataSource != null) - { - LodUtil.assertTrue(this.pos.equals(fullDataSource.getSectionPos()), "Attempting to cache a datasource with the wrong position. Meta file pos: [" + this.pos + "], data source pos: [" + fullDataSource.getSectionPos() + "]."); - } - - // save the updated data source - this.cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this, fullDataSource); - } - - // the task is complete - completionFuture.complete(fullDataSource); - - - if (this.needsUpdate) - { - // another update was requested while this update was being processed - if (cacheLoadingSource) - { - this.getOrLoadCachedDataSourceAsync(); - } - else - { - this.getDataSourceWithoutCachingAsync(); - } - } - }); - - return completionFuture; - } - - - - /** @return true if the queue was not empty and chunk data was applied to this meta file's {@link IFullDataSource}. */ - private boolean applyWriteQueueToFullDataSource(IFullDataSource fullDataSource) - { - // swap the write queue if it has queued chunks. - // Must be done in this order to ensure IWorldGenTaskTracker.isMemoryAddressValid() work properly. See IWorldGenTaskTracker.isMemoryAddressValid() for details. - boolean queueIsEmpty = this.writeQueueRef.get().queue.isEmpty(); - if (!queueIsEmpty) - { - this.swapWriteQueues(); - for (ChunkSizedFullDataAccessor chunk : this.backWriteQueue.queue) - { - fullDataSource.update(chunk); - } - - this.backWriteQueue.queue.clear(); - //LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count); - } - - return !queueIsEmpty || !this.doesDtoExist; - } - private void swapWriteQueues() - { - GuardedMultiAppendQueue writeQueue = this.writeQueueRef.getAndSet(this.backWriteQueue); - - // Acquire write lock and then release it again as we only need to ensure that the queue - // is not being appended to by another thread. Note that the above atomic swap & - // the guarantee that all append first acquire the appendLock means after the locK() call, - // there will be no other threads able to or is currently appending to the queue. - // Note: The above needs the getAndSet() to have at least Release Memory order. - // (not that java supports anything non volatile for getAndSet()...) - writeQueue.appendLock.writeLock().lock(); - writeQueue.appendLock.writeLock().unlock(); - - this.backWriteQueue = writeQueue; - } - - private void writeDataSource(IFullDataSource fullDataSource) - { - if (fullDataSource.isEmpty()) - { - // delete the empty data source - MetaDataDto dto = this.fullDataSourceProvider.getRepo().getByPrimaryKey(this.pos.serialize()); - if (dto != null) - { - this.fullDataSourceProvider.getRepo().delete(dto); - } - - this.doesDtoExist = false; - } - else - { - // update the data source and write the new data to file - - //LOGGER.info("Saving data file of {}", data.getSectionPos()); - try - { - // Write/Update data - LodUtil.assertTrue(this.baseMetaData != null); - - - // confirm the meta data properties are up to date // - - this.baseMetaData.dataDetailLevel = fullDataSource.getDataDetailLevel(); - this.fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getBinaryDataFormatVersion()); - LodUtil.assertTrue(this.fullDataSourceLoader != null, "No loader for " + fullDataSource.getClass() + " (v" + fullDataSource.getBinaryDataFormatVersion() + ")"); - - this.fullDataSourceClass = fullDataSource.getClass(); - this.baseMetaData.dataType = (this.fullDataSourceLoader == null) ? null : this.fullDataSourceLoader.datatype; - this.baseMetaData.binaryDataFormatVersion = fullDataSource.getBinaryDataFormatVersion(); - - - // save the data to the database // - super.writeToDatabase((bufferedOutputStream) -> fullDataSource.writeToStream((bufferedOutputStream), this.level), this.fullDataSourceProvider.getRepo()); - this.doesDtoExist = true; - } - catch (ClosedByInterruptException e) // thrown by buffers that are interrupted - { - // expected if the file handler is shut down, the exception can be ignored - //LOGGER.warn("FullData file writing interrupted.", e); - } - catch (IOException e) - { - LOGGER.error("Failed to save updated data for section " + this.pos, e); - } - } - } - - - - //================// - // helper classes // - //================// - - //TODO: use ConcurrentAppendSingleSwapContainer instead of below: - private static class GuardedMultiAppendQueue - { - ReentrantReadWriteLock appendLock = new ReentrantReadWriteLock(); - ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); - - } - - /** used to debug data source soft reference garbage collection */ - private static class DataObjTracker extends PhantomReference implements Closeable - { - public final DhSectionPos pos; - - - DataObjTracker(IFullDataSource data) - { - super(data, LIFE_CYCLE_DEBUG_QUEUE); - - if (LOG_DATA_SOURCE_LIVES) - { - //LOGGER.info("Phantom created on "+data.getSectionPos()+"! count: "+LIFE_CYCLE_DEBUG_SET.size()); - } - - LIFE_CYCLE_DEBUG_SET.add(this); - this.pos = data.getSectionPos(); - } - - @Override - public void close() { LIFE_CYCLE_DEBUG_SET.remove(this); } - - } - - /** used to debug data source soft reference garbage collection */ - private static class DataObjSoftTracker extends SoftReference implements Closeable - { - public final FullDataMetaFile file; - - - DataObjSoftTracker(FullDataMetaFile file, IFullDataSource data) - { - super(data, SOFT_REF_DEBUG_QUEUE); - SOFT_REF_DEBUG_SET.add(this); - this.file = file; - } - - @Override - public void close() { SOFT_REF_DEBUG_SET.remove(this); } - - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 1fc114b3e..7ec6d8872 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -20,9 +20,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.MissingWorldGenPositionFinder; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; @@ -35,7 +33,6 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.LodUtil; import org.apache.logging.log4j.Logger; -import javax.annotation.Nullable; import java.lang.ref.WeakReference; import java.util.*; import java.util.concurrent.*; @@ -46,7 +43,6 @@ import java.util.function.Function; public class GeneratedFullDataFileHandler extends FullDataFileHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final Timer CHUNK_GEN_FINISHED_TIMER = new Timer(); private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); @@ -55,6 +51,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler /** Used to prevent data sources from being garbage collected before their world gen finishes. */ private final ConcurrentHashMap generatingDataSourceByPos = new ConcurrentHashMap<>(); + + + //=============// + // constructor // + //=============// + public GeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } @@ -64,37 +66,18 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler //===========// @Override - public CompletableFuture readAsync(DhSectionPos pos) - { - CompletableFuture future = super.readAsync(pos); - return future.thenApply((dataSource) -> - { - // add world gen tasks for missing columns in the data source - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos); - if (worldGenQueue != null && metaFile != null) - { - this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource); - } - - return dataSource; - }); - } - - @Override - public void onRenderDataFileLoaded(DhSectionPos pos) + protected IFullDataSource get(DhSectionPos pos) { + IFullDataSource dataSource = super.get(pos); + // add world gen tasks for missing columns in the data source IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, false); - if (worldGenQueue != null && metaFile != null) + if (worldGenQueue != null) { - metaFile.getDataSourceWithoutCachingAsync().thenApply((fullDataSource) -> - { - this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, fullDataSource); - return fullDataSource; - }); + this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, pos, dataSource); } + + return dataSource; } @@ -111,27 +94,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler { boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); - LOGGER.info("Set world gen queue for level "+this.level+" to start."); - - this.ForEachFile(metaFile -> - { - IFullDataSource dataSource = metaFile.getCachedDataSourceNowOrNull(); - if (dataSource == null) - { - return; - } - - metaFile.genQueueChecked = false; // allow the system to check for missing positions again - this.queueWorldGenForMissingColumnsInDataSource(this.worldGenQueueRef.get(), metaFile, dataSource); - - if (dataSource instanceof CompleteFullDataSource) - { - return; - } - metaFile.markNeedsUpdate(); - }); - - this.flushAndSaveAsync(); // Trigger an update to the meta files + LOGGER.info("Set world gen queue for level ["+this.level+"]."); } public void clearGenerationQueue() @@ -140,14 +103,13 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler this.generatingDataSourceByPos.clear(); // clear the incomplete data sources } - // TODO what is this here for? + /** Can be used to remove positions that are outside the player's render distance. */ public void removeGenRequestIf(Function removeIf) { this.generatingDataSourceByPos.forEach((pos, dataSource) -> { if (removeIf.apply(pos)) { - //this.worldGenQueueRef.get().cancelGenTasks(pos); // shouldn't this be called if we actually want to stop world gen this.generatingDataSourceByPos.remove(pos); } }); @@ -160,104 +122,14 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler //=================// public void addWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.add(listener); } - public void removeWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.remove(listener); } - private IFullDataSource tryPromoteDataSource(IIncompleteFullDataSource source) - { - IFullDataSource newSource = source.tryPromotingToCompleteDataSource(); - if (newSource instanceof CompleteFullDataSource) - { - this.generatingDataSourceByPos.remove(source.getSectionPos()); - } - return newSource; - } - //========// // events // //========// - // Try update the gen queue on this data source. If null, then nothing was done. - @Nullable - private CompletableFuture updateFromExistingDataSourcesAsync(FullDataMetaFile file, IIncompleteFullDataSource data, boolean usePooledDataSources) - { - DhSectionPos pos = file.pos; - ArrayList existingFiles = new ArrayList<>(); - ArrayList missingPositions = new ArrayList<>(); - this.getDataFilesForPosition(pos, pos, existingFiles, missingPositions); - - if (missingPositions.size() == 1) - { - // Only missing myself. I.e. no child file data exists yet. - return this.tryStartGenTask(file, data); - } - else - { - // There are other data source files to sample from. - this.makeFiles(missingPositions, existingFiles); - return this.sampleFromFileArray(data, existingFiles, usePooledDataSources) - .thenApply(this::tryPromoteDataSource) - .exceptionally((e) -> - { - this.removeCorruptedFile(pos, e); - return null; - }); - } - } - @Nullable - private CompletableFuture tryStartGenTask(FullDataMetaFile metaFile, IIncompleteFullDataSource dataSource) // TODO after generation is finished, save and free any full datasources that aren't in use (IE high detail ones below the top) - { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null) - { - this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource); - return CompletableFuture.completedFuture(dataSource); - } - return null; - } - - @Override - public CompletableFuture onDataFileCreatedAsync(FullDataMetaFile file) - { - DhSectionPos pos = file.pos; - IIncompleteFullDataSource data = this.makeEmptyDataSource(pos); - CompletableFuture future = this.updateFromExistingDataSourcesAsync(file, data, true); - // Cant start gen task, so return the data - return future == null ? CompletableFuture.completedFuture(data) : future; - } - - @Override - public CompletableFuture onDataFileUpdateAsync(IFullDataSource fullDataSource, FullDataMetaFile file, boolean dataChanged) - { - LodUtil.assertTrue(this.fullDataRepo.existsWithPrimaryKey(file.pos.serialize()) || dataChanged); - - - if (fullDataSource instanceof CompleteFullDataSource) - { - this.generatingDataSourceByPos.remove(fullDataSource.getSectionPos()); - } - this.fireOnGenPosSuccessListeners(fullDataSource.getSectionPos()); - - - if (fullDataSource instanceof IIncompleteFullDataSource && !file.genQueueChecked) - { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null) - { - CompletableFuture future = this.updateFromExistingDataSourcesAsync(file, (IIncompleteFullDataSource) fullDataSource, false); - if (future != null) - { - final boolean finalDataChanged = dataChanged; - return future.thenApply((newSource) -> new DataFileUpdateResult(newSource, finalDataChanged)); - } - } - } - - return CompletableFuture.completedFuture(new DataFileUpdateResult(fullDataSource, dataChanged)); - } - private void onWorldGenTaskComplete(WorldGenResult genTaskResult, Throwable exception, GenTask genTask, DhSectionPos pos) { if (exception != null) @@ -270,18 +142,6 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } else if (genTaskResult.success) { - // generation completed, update the files and listener(s) - this.flushAndSaveAsync(pos).join(); - - // FIXME this is a bad fix to prevent full data sources saving incomplete, causing holes in the world after generation. - // The problem appears to be that the save may be happening too quickly, - // potentially happening before the meta file has the newly generated data added to it. - CHUNK_GEN_FINISHED_TIMER.schedule(new TimerTask() - { - @Override - public void run() { GeneratedFullDataFileHandler.this.flushAndSaveAsync(pos).join(); } - }, 4000L); - this.fireOnGenPosSuccessListeners(pos); return; } @@ -316,24 +176,9 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // helper methods // //================// - private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, FullDataMetaFile metaFile, IFullDataSource dataSource) + private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, DhSectionPos pos, IFullDataSource dataSource) { - // Due to a bug in the current system, some Complete data sources aren't actually complete - // and will need additional generation to finish - //if (dataSource instanceof CompleteFullDataSource) - //{ - // return; - //} - - if (metaFile.genQueueChecked) - { - // world gen has already been checked for this file - return; - } - metaFile.genQueueChecked = true; - - - // get the ungenerated pos list + // get the un-generated pos list byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); ArrayList genPosList = MissingWorldGenPositionFinder.getUngeneratedPosList(dataSource, minGeneratorSectionDetailLevel, true); @@ -341,17 +186,13 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler ArrayList> taskFutureList = new ArrayList<>(); for (DhSectionPos genPos : genPosList) { - // make sure each meta file has been created (not doing this will prevent down sampling and/or saving the generated data source) - this.getLoadOrMakeFile(genPos, true); - this.getLoadOrMakeFile(metaFile.pos, true); - // queue each gen task GenTask genTask = new GenTask(dataSource.getSectionPos(), new WeakReference<>(dataSource)); CompletableFuture worldGenFuture = worldGenQueue.submitGenTask(genPos, dataSource.getDataDetailLevel(), genTask); worldGenFuture.whenComplete((genTaskResult, ex) -> { this.onWorldGenTaskComplete(genTaskResult, ex, genTask, genPos); - this.onWorldGenTaskComplete(genTaskResult, ex, genTask, metaFile.pos); + this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos); }); taskFutureList.add(worldGenFuture); @@ -361,14 +202,13 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // mark the data source as generating if necessary if (taskFutureList.size() != 0) { - this.generatingDataSourceByPos.put(metaFile.pos, dataSource); + this.generatingDataSourceByPos.put(pos, dataSource); + CompletableFuture.allOf(taskFutureList.toArray(new CompletableFuture[0])) + .whenComplete((voidObj, ex) -> + { + this.generatingDataSourceByPos.remove(pos); + }); } - CompletableFuture.allOf(taskFutureList.toArray(new CompletableFuture[0])) - .whenComplete((voidObj, ex) -> - { - metaFile.flushAndSaveAsync(); - this.generatingDataSourceByPos.remove(metaFile.pos); - }); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index df6403cb4..aa7a29563 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -23,51 +23,19 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedF import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.FullDataRepo; -import org.jetbrains.annotations.Nullable; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; +/** + * Handles reading, writing, and updating {@link IFullDataSource}'s.
+ * Should be backed by a database handled by a {@link FullDataRepo}. + */ public interface IFullDataSourceProvider extends AutoCloseable { - CompletableFuture readAsync(DhSectionPos pos); - void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData); - CompletableFuture flushAndSaveAsync(); - CompletableFuture flushAndSaveAsync(DhSectionPos sectionPos); + CompletableFuture getAsync(DhSectionPos pos); - //long getCacheVersion(DhSectionPos sectionPos); - //boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion); - - CompletableFuture onDataFileCreatedAsync(FullDataMetaFile file); - default CompletableFuture onDataFileUpdateAsync(IFullDataSource fullDataSource, FullDataMetaFile file, boolean dataChanged) { return CompletableFuture.completedFuture(new DataFileUpdateResult(fullDataSource, dataChanged)); } - /** Can be used to update world gen queues or run any other data checking necessary when initially loading a file */ - default void onRenderDataFileLoaded(DhSectionPos pos) { } - - @Nullable - FullDataMetaFile getFileIfExist(DhSectionPos pos); + void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData); FullDataRepo getRepo(); - - - //================// - // helper classes // - //================// - - /** - * After a {@link FullDataMetaFile} has been updated the {@link IFullDataSourceProvider} may also need to modify it.
- * This specifically happens during world generation. - */ - class DataFileUpdateResult - { - IFullDataSource fullDataSource; - boolean dataSourceChanged; - - public DataFileUpdateResult(IFullDataSource fullDataSource, boolean dataSourceChanged) - { - this.fullDataSource = fullDataSource; - this.dataSourceChanged = dataSourceChanged; - } - } - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java index ddd724f4f..f383b5d41 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java @@ -142,6 +142,7 @@ public abstract class AbstractMetaDataContainerFile // helper classes // //================// + /** TODO replace with a method that accepts a {@link DhDataOutputStream} and writes to that instead */ @FunctionalInterface public interface IMetaDataWriterFunc { void writeBinaryDataToStream(T t) throws IOException; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java b/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java index b3851bbe9..465469508 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java @@ -35,6 +35,8 @@ public class BaseMetaData { public DhSectionPos pos; public int checksum; + /** @deprecated the database now has a last modified date time that should be used instead */ + @Deprecated public AtomicLong dataVersion = new AtomicLong(Long.MAX_VALUE); public byte dataDetailLevel; public EDhApiWorldGenerationStep worldGenStep; @@ -46,11 +48,10 @@ public class BaseMetaData - public BaseMetaData(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, long dataVersion) + public BaseMetaData(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion) { this.pos = pos; this.checksum = checksum; - this.dataVersion = new AtomicLong(dataVersion); this.dataDetailLevel = dataDetailLevel; this.worldGenStep = worldGenStep; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java index 7ec584743..061889f51 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java @@ -23,8 +23,6 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.DataSourceReferenceTracker; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; @@ -38,13 +36,13 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.sql.MetaDataDto; import com.seibel.distanthorizons.core.util.AtomicsUtil; import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.Reference; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.awt.*; import java.io.*; +import java.lang.ref.SoftReference; import java.util.Random; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; @@ -64,7 +62,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements * When clearing, don't set to null, instead create a SoftReference containing null. * This makes null checks simpler. */ - private DataSourceReferenceTracker.RenderDataSourceSoftRef cachedRenderDataSourceRef = new DataSourceReferenceTracker.RenderDataSourceSoftRef(this, null); + private SoftReference cachedRenderDataSourceRef = new SoftReference<>(null); private final AtomicReference> renderSourceLoadFutureRef = new AtomicReference<>(null); private final IDhClientLevel clientLevel; @@ -109,9 +107,6 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements LodUtil.assertTrue(this.baseMetaData != null); this.doesDtoExist = true; DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus); - - // handles world gen queuing for missing columns - this.fullDataSourceProvider.onRenderDataFileLoaded(this.baseMetaData.pos); } @@ -196,11 +191,11 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements this.baseMetaData = new BaseMetaData( newColumnRenderSource.getSectionPos(), -1, newColumnRenderSource.getDataDetailLevel(), newColumnRenderSource.worldGenStep, RENDER_SOURCE_TYPE, - newColumnRenderSource.getRenderDataFormatVersion(), Long.MAX_VALUE); + newColumnRenderSource.getRenderDataFormatVersion()); this.updateRenderCacheAsync(newColumnRenderSource).whenComplete((voidObj, ex) -> { - this.cachedRenderDataSourceRef = new DataSourceReferenceTracker.RenderDataSourceSoftRef(this, newColumnRenderSource); + this.cachedRenderDataSourceRef = new SoftReference<>(newColumnRenderSource); this.renderSourceLoadFutureRef.set(null); getSourceFuture.complete(newColumnRenderSource); @@ -248,7 +243,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements this.renderSourceLoadFutureRef.set(null); - this.cachedRenderDataSourceRef = new DataSourceReferenceTracker.RenderDataSourceSoftRef(this, renderSource); + this.cachedRenderDataSourceRef = new SoftReference<>(renderSource); getSourceFuture.complete(renderSource); }); } @@ -273,32 +268,30 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements DebugRenderer.BoxWithLife debugBox = new DebugRenderer.BoxWithLife(new DebugRenderer.Box(renderSource.sectionPos, 74f, 86f, 0.1f, Color.red), 1.0, 32f, Color.green.darker()); - // Skip updating the cache if the data file is already up-to-date - FullDataMetaFile dataFile = this.fullDataSourceProvider.getFileIfExist(this.pos); - if (!ALWAYS_INVALIDATE_CACHE && dataFile != null && dataFile.baseMetaData != null && dataFile.baseMetaData.checksum == this.baseMetaData.dataVersion.get()) // TODO can we make it so the version comparisons either both use the checksum or the dataVersion? Comparing checksum and dataVersion is kinda confusing - { - LOGGER.debug("Skipping render cache update for " + this.pos); - renderSource.localVersion.incrementAndGet(); - return CompletableFuture.completedFuture(renderSource); - } + //// Skip updating the cache if the data file is already up-to-date + //FullDataMetaFile dataFile = this.fullDataSourceProvider.getDtoIfExist(this.pos); + //if (!ALWAYS_INVALIDATE_CACHE && dataFile != null && dataFile.baseMetaData != null && dataFile.baseMetaData.checksum == this.baseMetaData.dataVersion.get()) // TODO can we make it so the version comparisons either both use the checksum or the dataVersion? Comparing checksum and dataVersion is kinda confusing + //{ + // LOGGER.debug("Skipping render cache update for " + this.pos); + // renderSource.localVersion.incrementAndGet(); + // return CompletableFuture.completedFuture(renderSource); + //} - final Reference renderDataVersionRef = new Reference<>(Integer.MAX_VALUE); - // get the full data source CompletableFuture fullDataSourceFuture = - this.fullDataSourceProvider.readAsync(renderSource.getSectionPos()) + this.fullDataSourceProvider.getAsync(renderSource.getSectionPos()) .thenApply((fullDataSource) -> { debugBox.box.color = Color.yellow.darker(); - // get the metaFile's version - FullDataMetaFile renderSourceMetaFile = this.fullDataSourceProvider.getFileIfExist(this.pos); - if (renderSourceMetaFile != null && renderSourceMetaFile.baseMetaData != null) - { - renderDataVersionRef.value = renderSourceMetaFile.baseMetaData.checksum; - } + //// get the metaFile's version + //FullDataMetaFile renderSourceMetaFile = this.fullDataSourceProvider.getDtoIfExist(this.pos); + //if (renderSourceMetaFile != null && renderSourceMetaFile.baseMetaData != null) + //{ + // renderDataVersionRef.value = renderSourceMetaFile.baseMetaData.checksum; + //} return fullDataSource; }).exceptionally((ex) -> @@ -332,7 +325,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements renderSource.updateFromRenderSource(newRenderSource); // update the meta data - this.baseMetaData.dataVersion.set(renderDataVersionRef.value); + this.baseMetaData.dataVersion.set(Integer.MAX_VALUE); this.baseMetaData.dataDetailLevel = renderSource.getDataDetailLevel(); this.baseMetaData.dataType = RENDER_SOURCE_TYPE; this.baseMetaData.binaryDataFormatVersion = renderSource.getRenderDataFormatVersion(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index d417929e6..1e36d896f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -213,7 +213,7 @@ public class RenderSourceFileHandler implements IRenderSourceProvider { // convert to the lowest detail level so all detail levels are updated this.writeChunkDataToFileRecursively(chunkDataView, DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - this.fullDataSourceProvider.writeChunkDataToFile(sectionPos, chunkDataView); + this.fullDataSourceProvider.updateDataSourcesWithChunkData(chunkDataView); } private void writeChunkDataToFileRecursively(ChunkSizedFullDataAccessor chunk, byte sectionDetailLevel) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 2b087059a..85fdb0c58 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -219,7 +219,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable { // get the data source to compare against IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), this.currentClientLevel, testLevelFolder, false); - IFullDataSource testFullDataSource = tempLevel.getFileHandler().readAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); + IFullDataSource testFullDataSource = tempLevel.getFileHandler().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); if (testFullDataSource == null) { continue; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java index 48754664a..af9c28755 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java @@ -35,7 +35,6 @@ public interface IWorldGenerationQueue extends Closeable byte highestDataDetail(); CompletableFuture submitGenTask(DhSectionPos pos, byte requiredDataDetail, IWorldGenTaskTracker tracker); - void cancelGenTasks(Iterable positions); /** @param targetPos the position that world generation should be centered around, generally this will be the player's position. */ void startGenerationQueueAndSetTargetPos(DhBlockPos2D targetPos); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 00fd702f0..7c3b68744 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -159,12 +159,6 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender return future; } - @Override - public void cancelGenTasks(Iterable positions) - { - // TODO Should we cancel generation of chunks that were loaded by the player? - } - //===============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 0bc03ace4..2fae17053 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -189,20 +189,7 @@ public class ClientLevelModule implements Closeable } else { - this.parentClientLevel.getFileHandler().writeChunkDataToFile(pos, data); - } - } - - public CompletableFuture saveAsync() - { - ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); - if (ClientRenderState != null) - { - return ClientRenderState.renderSourceFileHandler.flushAndSaveAsync(); - } - else - { - return CompletableFuture.completedFuture(null); + this.parentClientLevel.getFileHandler().updateDataSourcesWithChunkData(data); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 6edb8640e..545cee3e1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -115,12 +115,6 @@ public class DhClientLevel extends DhLevel implements IDhClientLevel @Override public ILevelWrapper getLevelWrapper() { return levelWrapper; } - @Override - public CompletableFuture saveAsync() - { - return CompletableFuture.allOf(clientside.saveAsync(), dataFileHandler.flushAndSaveAsync()); - } - @Override public void saveWrites(ChunkSizedFullDataAccessor data) { this.clientside.writeChunkDataToFile(data); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index 1d10bbe84..913bbe80f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -22,12 +22,14 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.render.LodRenderSection; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; @@ -39,7 +41,7 @@ import com.seibel.distanthorizons.coreapi.util.math.Mat4f; import org.apache.logging.log4j.Logger; import java.awt.*; -import java.util.concurrent.CompletableFuture; +import java.util.Iterator; /** The level used on a singleplayer world */ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhServerLevel @@ -90,25 +92,37 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS @Override public void doWorldGen() { - this.serverside.worldGeneratorEnabledConfig.pollNewValue(); + this.serverside.worldGeneratorEnabledConfig.pollNewValue(); // if not called the get() line below may not boolean shouldDoWorldGen = this.serverside.worldGeneratorEnabledConfig.get() && this.clientside.isRendering(); boolean isWorldGenRunning = this.serverside.worldGenModule.isWorldGenRunning(); if (shouldDoWorldGen && !isWorldGenRunning) { // start world gen + + // create a new queue this.serverside.worldGenModule.startWorldGen(this.serverside.dataFileHandler, new ServerLevelModule.WorldGenState(this)); + + // populate the queue based on the current rendering tree + ClientLevelModule.ClientRenderState renderState = this.clientside.ClientRenderStateRef.get(); + Iterator> iterator = renderState.quadtree.leafNodeIterator(); + while (iterator.hasNext()) + { + QuadNode node = iterator.next(); + this.serverside.dataFileHandler.getAsync(node.sectionPos); + } } else if (!shouldDoWorldGen && isWorldGenRunning) { // stop world gen this.serverside.worldGenModule.stopWorldGen(this.serverside.dataFileHandler); } - - if (this.serverside.worldGenModule.isWorldGenRunning()) + + if (isWorldGenRunning) { ClientLevelModule.ClientRenderState renderState = this.clientside.ClientRenderStateRef.get(); if (renderState != null && renderState.quadtree != null) { + // remove any generator sections that are out of bounds this.serverside.dataFileHandler.removeGenRequestIf(pos -> !renderState.quadtree.isSectionPosInBounds(pos)); } @@ -177,11 +191,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } - @Override - public CompletableFuture saveAsync() - { - return CompletableFuture.allOf(clientside.saveAsync(), getFileHandler().flushAndSaveAsync()); - } + //===============// // data handling // diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index fdd34567c..a568d6a44 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -30,8 +30,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; import org.apache.logging.log4j.Logger; -import java.util.concurrent.CompletableFuture; - public class DhServerLevel extends DhLevel implements IDhServerLevel { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -56,7 +54,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel { DhSectionPos pos = data.getSectionPos(); pos = pos.convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); - this.getFileHandler().writeChunkDataToFile(pos, data); + this.getFileHandler().updateDataSourcesWithChunkData(data); } @Override @@ -70,9 +68,6 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel LOGGER.info("Closed DHLevel for {}", getLevelWrapper()); } - @Override - public CompletableFuture saveAsync() { return getFileHandler().flushAndSaveAsync(); } - @Override public void doWorldGen() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java index 15343312b..1ae9d402b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java @@ -29,7 +29,6 @@ import java.util.concurrent.CompletableFuture; public interface IDhLevel extends AutoCloseable { int getMinY(); - CompletableFuture saveAsync(); /** * May return either a client or server level wrapper.
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java index 86f9fc34b..a79ee19cb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java @@ -41,6 +41,10 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo public String getPrimaryKeyName() { return "DhSectionPos"; } + //=======================// + // repo required methods // + //=======================// + @Override public MetaDataDto convertDictionaryToDto(Map objectMap) throws ClassCastException { @@ -59,7 +63,7 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo BaseMetaData baseMetaData = new BaseMetaData(pos, checksum, dataDetailLevel, worldGenStep, - dataType, binaryDataFormatVersion, dataVersion); + dataType, binaryDataFormatVersion); // binary data byte[] dataByteArray = (byte[]) objectMap.get("Data"); @@ -137,4 +141,30 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo } + + //=====================// + // data source methods // + //=====================// + + /** + * Returns the highest numerical detail level in this table.
+ * Returns {@link DhSectionPos#SECTION_MINIMUM_DETAIL_LEVEL} if no data is present. + */ + public int getMaxSectionDetailLevel() + { + Map resultMap = this.queryDictionaryFirst("select MAX(DataDetailLevel) as maxDetailLevel from DhFullData;"); + int maxDetailLevel; + if (resultMap == null || resultMap.get("maxDetailLevel") == null) + { + maxDetailLevel = 0; + } + else + { + maxDetailLevel = (int)resultMap.get("maxDetailLevel"); + } + + return maxDetailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + } + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java index 31d2fe96e..bdd1b5566 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java @@ -21,7 +21,13 @@ package com.seibel.distanthorizons.core.sql; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; /** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */ public class MetaDataDto implements IBaseDTO @@ -40,4 +46,12 @@ public class MetaDataDto implements IBaseDTO @Override public String getPrimaryKeyString() { return this.baseMetaData.pos.serialize(); } + /** @return a stream for the data contained in this DTO. */ + public DhDataInputStream getInputStream() throws IOException + { + InputStream inputStream = new ByteArrayInputStream(this.dataArray); + DhDataInputStream compressedStream = new DhDataInputStream(inputStream); + return compressedStream; + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java index ef59e1151..275d606ea 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java @@ -151,15 +151,10 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor public void doWorldGen() { this.dhLevels.forEach(DhClientServerLevel::doWorldGen); } - @Override - public CompletableFuture saveAndFlush() { return CompletableFuture.allOf(this.dhLevels.stream().map(DhClientServerLevel::saveAsync).toArray(CompletableFuture[]::new)); } - /** synchronized to prevent a rare issue where the server tries closing the same world multiple times in rapid succession. */ @Override public synchronized void close() { - // at this point the levels are probably unloaded, so this save call usually generally won't do anything - this.saveAndFlush(); this.f3Message.close(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java index 7aa0e7a64..b68cc9936 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java @@ -173,12 +173,6 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld // Not implemented } - @Override - public CompletableFuture saveAndFlush() - { - return CompletableFuture.allOf(this.levels.values().stream().map(DhClientLevel::saveAsync).toArray(CompletableFuture[]::new)); - } - @Override public void close() { @@ -188,7 +182,6 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld // } - this.saveAndFlush(); for (DhClientLevel dhClientLevel : this.levels.values()) { LOGGER.info("Unloading level " + dhClientLevel.getLevelWrapper().getDimensionType().getDimensionName()); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java index 4d35d79b8..c306e2b04 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java @@ -188,12 +188,6 @@ public class DhServerWorld extends AbstractDhWorld implements IDhServerWorld public void doWorldGen() { this.levels.values().forEach(DhServerLevel::doWorldGen); } - @Override - public CompletableFuture saveAndFlush() - { - return CompletableFuture.allOf(this.levels.values().stream().map(DhServerLevel::saveAsync).toArray(CompletableFuture[]::new)); - } - @Override public void close() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/IDhWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/IDhWorld.java index dd631015c..9334ccbbd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/IDhWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/IDhWorld.java @@ -34,6 +34,4 @@ public interface IDhWorld void unloadLevel(@NotNull ILevelWrapper levelWrapper); - CompletableFuture saveAndFlush(); - }