wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Overhaul and simplify FullDataFileHandler

Browse Source
This commit is contained in:
James Seibel
2024-01-06 14:02:45 -06:00
parent ab031d2961
commit ec29ea8cc1
36 changed files with 549 additions and 1775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/Initializer.java
-3
View File
@@ -20,7 +20,6 @@
package com.seibel.distanthorizons.core;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.file.DataSourceReferenceTracker;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.coreapi.ModInfo;
import com.seibel.distanthorizons.core.world.DhApiWorldProxy;
@@ -75,8 +74,6 @@ public class Initializer
LOGGER.error("Programmer Error: No ["+IWrapperFactory.class.getSimpleName()+"] assigned to the DhApi.");
}
DataSourceReferenceTracker.startGarbageCollectorBackgroundThread();
}
}
core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java
Vendored
+1 -1
View File
@@ -212,7 +212,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo
try
{
// attempt to get/generate the data source for this section
IFullDataSource dataSource = level.getFileHandler().readAsync(sectionPos).get();
IFullDataSource dataSource = level.getFileHandler().getAsync(sectionPos).get();
if (dataSource == null)
{
return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + sectionPos + "].");
core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java
-12
View File
@@ -139,18 +139,6 @@ public class ServerApi
}
}
@Deprecated // TODO not implemented, remove
public void serverSaveEvent()
{
LOGGER.debug("Server world " + SharedApi.getAbstractDhWorld() + " saving");
AbstractDhWorld serverWorld = SharedApi.getAbstractDhWorld();
if (serverWorld != null)
{
serverWorld.saveAndFlush();
}
}
//=======================//
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java
+2 -2
View File
@@ -56,7 +56,7 @@ public class FullDataDownSampler
{
for (int zOffset = 0; zOffset < sectionSizeNeeded; zOffset++)
{
CompletableFuture<IFullDataSource> future = provider.readAsync(new DhSectionPos(
CompletableFuture<IFullDataSource> future = provider.getAsync(new DhSectionPos(
CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset, basePos.z + zOffset));
future = future.whenComplete((source, ex) -> {
if (ex == null && source != null && source instanceof CompleteFullDataSource)
@@ -80,7 +80,7 @@ public class FullDataDownSampler
{
for (int zOffset = 0; zOffset < CompleteFullDataSource.WIDTH; zOffset++)
{
CompletableFuture<IFullDataSource> future = provider.readAsync(new DhSectionPos(
CompletableFuture<IFullDataSource> future = provider.getAsync(new DhSectionPos(
CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset * multiplier, basePos.z + zOffset * multiplier));
future = future.whenComplete((source, ex) -> {
if (ex == null && source != null && source instanceof CompleteFullDataSource)
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java
+20 -21
View File
@@ -21,10 +21,9 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.loader;
import com.google.common.collect.HashMultimap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import java.io.IOException;
import java.util.*;
@@ -120,31 +119,31 @@ public abstract class AbstractFullDataSourceLoader
// data loading //
//==============//
/** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */
public IFullDataSource loadTemporaryDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException
{
IFullDataSource dataSource = this.tryGetPooledSource();
if (dataSource != null)
{
dataSource.repopulateFromStream(dto, dto.getInputStream(), level);
}
else
{
dataSource = this.loadDataSource(dto, level);
}
return dataSource;
}
/**
* Can return null if any of the requirements aren't met.
*
* @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down
*/
public IFullDataSource loadDataSource(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
public IFullDataSource loadDataSource(MetaDataDto dto, IDhLevel level) throws IOException, InterruptedException
{
IFullDataSource dataSource = this.createEmptyDataSource(dataFile.pos);
dataSource.populateFromStream(dataFile, inputStream, level);
return dataSource;
}
/** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */
public IFullDataSource loadTemporaryDataSource(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
IFullDataSource dataSource = this.tryGetPooledSource();
if (dataSource != null)
{
dataSource.repopulateFromStream(dataFile, inputStream, level);
}
else
{
dataSource = this.loadDataSource(dataFile, inputStream, level);
}
IFullDataSource dataSource = this.createEmptyDataSource(dto.baseMetaData.pos);
dataSource.populateFromStream(dto, dto.getInputStream(), level);
return dataSource;
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java
+1 -1
View File
@@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFull
public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader
{
public CompleteFullDataSourceLoader() { super(CompleteFullDataSource.class, CompleteFullDataSource.DATA_SOURCE_TYPE, new byte[]{CompleteFullDataSource.DATA_FORMAT_VERSION}); }
public CompleteFullDataSourceLoader() { super(CompleteFullDataSource.class, CompleteFullDataSource.DATA_TYPE_NAME, new byte[]{CompleteFullDataSource.DATA_FORMAT_VERSION}); }
@Override
protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return CompleteFullDataSource.createEmpty(pos); }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java
+1 -1
View File
@@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIn
public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader
{
public HighDetailIncompleteFullDataSourceLoader() { super(HighDetailIncompleteFullDataSource.class, HighDetailIncompleteFullDataSource.DATA_SOURCE_TYPE, new byte[]{HighDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); }
public HighDetailIncompleteFullDataSourceLoader() { super(HighDetailIncompleteFullDataSource.class, HighDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{HighDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); }
@Override
protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return HighDetailIncompleteFullDataSource.createEmpty(pos); }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java
+1 -1
View File
@@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailInc
public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader
{
public LowDetailIncompleteFullDataSourceLoader() { super(LowDetailIncompleteFullDataSource.class, LowDetailIncompleteFullDataSource.DATA_SOURCE_TYPE, new byte[]{LowDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); }
public LowDetailIncompleteFullDataSourceLoader() { super(LowDetailIncompleteFullDataSource.class, LowDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{LowDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); }
@Override
protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return LowDetailIncompleteFullDataSource.createEmpty(pos); }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java
+9 -7
View File
@@ -25,12 +25,12 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArr
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
@@ -59,7 +59,9 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
public static final byte DATA_FORMAT_VERSION = 3;
public static final String DATA_SOURCE_TYPE = "CompleteFullDataSource";
public static final String DATA_TYPE_NAME = "CompleteFullDataSource";
@Override
public String getDataTypeName() { return DATA_TYPE_NAME; }
private DhSectionPos sectionPos;
private boolean isEmpty = true;
@@ -103,12 +105,12 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataDetail = inputStream.readInt();
if (dataDetail != dataFile.baseMetaData.dataDetailLevel)
if (dataDetail != dto.baseMetaData.dataDetailLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetail + " != " + dataFile.baseMetaData.dataDetailLevel));
throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetail + " != " + dto.baseMetaData.dataDetailLevel));
}
int width = inputStream.readInt();
@@ -185,7 +187,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
return true;
}
@Override
public long[][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream dataInputStream) throws IOException
public long[][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream dataInputStream) throws IOException
{
// Data array length
int dataPresentFlag = dataInputStream.readInt();
@@ -428,7 +430,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); }
@Override
public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; }
public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; }
@Override
public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java
+12 -10
View File
@@ -26,11 +26,11 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColum
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
@@ -72,7 +72,9 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
public static final byte MAX_SECTION_DETAIL = SECTION_SIZE_OFFSET + SPARSE_UNIT_DETAIL;
public static final byte DATA_FORMAT_VERSION = 3;
public static final String DATA_SOURCE_TYPE = "HighDetailIncompleteFullDataSource";
public static final String DATA_TYPE_NAME = "HighDetailIncompleteFullDataSource";
@Override
public String getDataTypeName() { return DATA_TYPE_NAME; }
protected final FullDataPointIdMap mapping;
@@ -141,15 +143,15 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
LodUtil.assertTrue(dataFile.pos.getDetailLevel() > SPARSE_UNIT_DETAIL);
LodUtil.assertTrue(dataFile.pos.getDetailLevel() <= MAX_SECTION_DETAIL);
LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() > SPARSE_UNIT_DETAIL);
LodUtil.assertTrue(dto.baseMetaData.pos.getDetailLevel() <= MAX_SECTION_DETAIL);
int dataDetailLevel = inputStream.readShort();
if (dataDetailLevel != dataFile.baseMetaData.dataDetailLevel)
if (dataDetailLevel != dto.baseMetaData.dataDetailLevel)
{
throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dataFile.baseMetaData.dataDetailLevel+"]");
throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.baseMetaData.dataDetailLevel+"]");
}
// confirm that the detail level is correct
@@ -254,11 +256,11 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
return true;
}
@Override
public long[][][] readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException
public long[][][] readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException
{
// calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize
// TODO these values should be constant, should we still be calculating them like this?
int chunks = BitShiftUtil.powerOfTwo(dataFile.pos.getDetailLevel() - SPARSE_UNIT_DETAIL);
int chunks = BitShiftUtil.powerOfTwo(dto.baseMetaData.pos.getDetailLevel() - SPARSE_UNIT_DETAIL);
int dataPointsPerChunk = SECTION_SIZE / chunks;
@@ -472,7 +474,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); }
@Override
public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; }
public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; }
@Override
public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java
+9 -7
View File
@@ -26,11 +26,11 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColum
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
@@ -63,7 +63,9 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
public static final byte DATA_FORMAT_VERSION = 3;
public static final String DATA_SOURCE_TYPE = "LowDetailIncompleteFullDataSource";
public static final String DATA_TYPE_NAME = "LowDetailIncompleteFullDataSource";
@Override
public String getDataTypeName() { return DATA_TYPE_NAME; }
private DhSectionPos sectionPos;
@@ -118,12 +120,12 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
}
@Override
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataDetailLevel = inputStream.readInt();
if (dataDetailLevel != dataFile.baseMetaData.dataDetailLevel)
if (dataDetailLevel != dto.baseMetaData.dataDetailLevel)
{
throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dataFile.baseMetaData.dataDetailLevel));
throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.baseMetaData.dataDetailLevel));
}
int width = inputStream.readInt();
@@ -186,7 +188,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
return true;
}
@Override
public StreamDataPointContainer readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException
public StreamDataPointContainer readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException
{
// is source empty flag
int dataPresentFlag = inputStream.readInt();
@@ -320,7 +322,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
@Override
public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); }
@Override
public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; }
public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; }
@Override
public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java
+15 -10
View File
@@ -20,6 +20,7 @@
package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
@@ -27,15 +28,12 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedF
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.IFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import org.jetbrains.annotations.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
/**
* Base for all Full Data Source objects. <br><br>
@@ -61,9 +59,17 @@ public interface IFullDataSource
DhSectionPos getSectionPos();
/**
* Returns the name of this data source. <br>
* Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse
* the binary data when read from file.
*/
String getDataTypeName();
/** Returns the detail level of the data contained by this {@link IFullDataSource}. */
byte getDataDetailLevel();
byte getBinaryDataFormatVersion();
/** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */
byte getDataFormatVersion();
EDhApiWorldGenerationStep getWorldGenStep();
void update(ChunkSizedFullDataAccessor data);
@@ -101,7 +107,6 @@ public interface IFullDataSource
// basic stream handling //
//=======================//
// TODO make this blow up in IStreamableFullDataSource instead of the children
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
*
@@ -112,15 +117,15 @@ public interface IFullDataSource
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
*
* @see IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel)
* @see IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel)
*/
void populateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
void populateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
/**
* Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies.
*
* @see IStreamableFullDataSource#repopulateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel)
* @see IStreamableFullDataSource#repopulateFromStream(MetaDataDto, DhDataInputStream, IDhLevel)
*/
void repopulateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
void repopulateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException;
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java
+13 -13
View File
@@ -22,8 +22,8 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
@@ -39,7 +39,7 @@ import java.io.IOException;
*
* @param <SummaryDataType> defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}.
* @param <DataContainerType> defines the object holding the data points, probably long[][] or long[][][].
* {@link IStreamableFullDataSource#populateFromStream(FullDataMetaFile, DhDataInputStream, IDhLevel) populateFromStream}
* {@link IStreamableFullDataSource#populateFromStream(MetaDataDto, DhDataInputStream, IDhLevel) populateFromStream}
* for the full reasoning.
*/
public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFullDataSource.FullDataSourceSummaryData, DataContainerType> extends IFullDataSource
@@ -56,14 +56,14 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
* @see IStreamableFullDataSource#populateFromStream
*/
@Override
default void repopulateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
default void repopulateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
// clear/overwrite the old data
this.resizeDataStructuresForRepopulation(dataFile.pos);
this.getMapping().clear(dataFile.pos);
this.resizeDataStructuresForRepopulation(dto.baseMetaData.pos);
this.getMapping().clear(dto.baseMetaData.pos);
// set the new data
this.populateFromStream(dataFile, inputStream, level);
this.populateFromStream(dto, inputStream, level);
}
/**
@@ -71,13 +71,13 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
* This is expected to be used with an empty {@link IStreamableFullDataSource} and functions similar to a constructor.
*/
@Override
default void populateFromStream(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
default void populateFromStream(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
SummaryDataType summaryData = this.readSourceSummaryInfo(dataFile, inputStream, level);
SummaryDataType summaryData = this.readSourceSummaryInfo(dto, inputStream, level);
this.setSourceSummaryData(summaryData);
DataContainerType dataPoints = this.readDataPoints(dataFile, summaryData.dataWidth, inputStream);
DataContainerType dataPoints = this.readDataPoints(dto, summaryData.dataWidth, inputStream);
if (dataPoints == null)
{
return;
@@ -114,19 +114,19 @@ public interface IStreamableFullDataSource<SummaryDataType extends IStreamableFu
*/
void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException;
/**
* Confirms that the given {@link FullDataMetaFile} is valid for this {@link IStreamableFullDataSource}. <br>
* Confirms that the given {@link MetaDataDto} is valid for this {@link IStreamableFullDataSource}. <br>
* This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed.
*
* @throws IOException if the {@link FullDataMetaFile} isn't valid for this object.
* @throws IOException if the {@link MetaDataDto} isn't valid for this object.
*/
SummaryDataType readSourceSummaryInfo(FullDataMetaFile dataFile, DhDataInputStream inputStream, IDhLevel level) throws IOException;
SummaryDataType readSourceSummaryInfo(MetaDataDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException;
void setSourceSummaryData(SummaryDataType summaryData);
/** @return true if any data points were present and written, false if this object was empty */
boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException;
/** @return null if no data points were present */
DataContainerType readDataPoints(FullDataMetaFile dataFile, int width, DhDataInputStream inputStream) throws IOException;
DataContainerType readDataPoints(MetaDataDto dto, int width, DhDataInputStream inputStream) throws IOException;
void setDataPoints(DataContainerType dataPoints);
core/src/main/java/com/seibel/distanthorizons/core/file/DataSourceReferenceTracker.java
-233
View File
@@ -1,233 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.renderfile.RenderDataMetaFile;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import org.apache.logging.log4j.Logger;
import java.io.Closeable;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Keeps track of {@link FullDataMetaFile} and {@link RenderDataMetaFile}'s
* and handles freeing their underlying data sources if they go unused for a certain amount of time.
*/
public class DataSourceReferenceTracker
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final boolean LOG_GARBAGE_COLLECTIONS = false;
/** How often the garbage collector thread will run */
private static final long MS_BETWEEN_GARBAGE_CHECKS = TimeUnit.SECONDS.toMillis(60);
/** How long a data source has to go unused before it can be freed */
private static final long MS_TO_EXPIRE_DATA_SOURCE = TimeUnit.SECONDS.toMillis(60);
// these queues are populated by the JVM's garbage collector after the assigned soft reference is freed
private static final ReferenceQueue<IFullDataSource> FULL_DATA_GARBAGE_COLLECTED_QUEUE = new ReferenceQueue<>();
private static final ReferenceQueue<ColumnRenderSource> RENDER_DATA_GARBAGE_COLLECTED_QUEUE = new ReferenceQueue<>();
// TODO using a ConcurrentHashMap may or may not be the best choice here
private static final Set<FullDataSourceSoftRef> FULL_DATA_SOFT_REFS = ConcurrentHashMap.newKeySet();
private static final Set<RenderDataSourceSoftRef> RENDER_DATA_SOFT_REFS = ConcurrentHashMap.newKeySet();
private static final ThreadPoolExecutor GARBAGE_COLLECTOR_THREAD = ThreadUtil.makeSingleThreadPool("DataSourceReferenceTracker", ThreadUtil.MINIMUM_RELATIVE_PRIORITY);
//=================//
// collector logic //
//=================//
/** Warning: this should not be called more than once. */
public static void startGarbageCollectorBackgroundThread() { /*GARBAGE_COLLECTOR_THREAD.execute(() -> garbageCollectorLoop());*/ }
private static void garbageCollectorLoop()
{
while(true)
{
try
{
runGarbageCollection();
Thread.sleep(MS_BETWEEN_GARBAGE_CHECKS);
}
catch (InterruptedException e)
{
LOGGER.error("Garbage collector thread interrupted.", e);
}
catch (Exception e)
{
LOGGER.error("Unexpected data source garbage collector exception: " + e.getMessage(), e);
}
}
}
public static void runGarbageCollection()
{
removeGarbageCollectedDataSources();
removeExpiredDataSources();
}
private static void removeGarbageCollectedDataSources()
{
FullDataSourceSoftRef garbageCollectedFullDataSoftRef = (FullDataSourceSoftRef) FULL_DATA_GARBAGE_COLLECTED_QUEUE.poll();
while (garbageCollectedFullDataSoftRef != null)
{
if (LOG_GARBAGE_COLLECTIONS)
{
LOGGER.info("Full Data at pos: " + garbageCollectedFullDataSoftRef.metaFile.pos + " has been soft released.");
}
garbageCollectedFullDataSoftRef.close();
garbageCollectedFullDataSoftRef = (FullDataSourceSoftRef) FULL_DATA_GARBAGE_COLLECTED_QUEUE.poll();
}
RenderDataSourceSoftRef renderSoftRef = (RenderDataSourceSoftRef) RENDER_DATA_GARBAGE_COLLECTED_QUEUE.poll();
while (renderSoftRef != null)
{
if (LOG_GARBAGE_COLLECTIONS)
{
LOGGER.info("Render Data at pos: " + renderSoftRef.metaFile.pos + " has been soft released.");
}
renderSoftRef.close();
renderSoftRef = (RenderDataSourceSoftRef) RENDER_DATA_GARBAGE_COLLECTED_QUEUE.poll();
}
}
private static void removeExpiredDataSources()
{
// TODO merge these loops
FULL_DATA_SOFT_REFS.removeIf((fullDataSoftRef) ->
{
boolean remove = fullDataSoftRef.isDataSourceExpired() || (fullDataSoftRef.silentGet() == null);
if (remove)
{
fullDataSoftRef.clear();
fullDataSoftRef.close();
if (LOG_GARBAGE_COLLECTIONS)
{
LOGGER.info("Full Data at pos: " + fullDataSoftRef.metaFile.pos + " has expired and will be released at the next GC. ["+FULL_DATA_SOFT_REFS.size()+"] Full data sources remain.");
}
}
return remove;
});
// TODO merge these loops
RENDER_DATA_SOFT_REFS.removeIf((renderDataSoftRef) ->
{
boolean remove = renderDataSoftRef.isDataSourceExpired() || (renderDataSoftRef.silentGet() == null);
if (remove)
{
renderDataSoftRef.clear();
renderDataSoftRef.close();
if (LOG_GARBAGE_COLLECTIONS)
{
LOGGER.info("Render Data at pos: " + renderDataSoftRef.metaFile.pos + " has expired and will be released at the next GC. ["+RENDER_DATA_SOFT_REFS.size()+"] Render data sources remain.");
}
}
return remove;
});
}
//================//
// helper classes //
//================//
public static class FullDataSourceSoftRef extends AbstractDataSourceSoftTracker<FullDataMetaFile, IFullDataSource>
{
public FullDataSourceSoftRef(FullDataMetaFile metaFile, IFullDataSource data)
{
super(metaFile, data, FULL_DATA_GARBAGE_COLLECTED_QUEUE);
FULL_DATA_SOFT_REFS.add(this);
}
@Override
public void close() { FULL_DATA_SOFT_REFS.remove(this); }
}
public static class RenderDataSourceSoftRef extends AbstractDataSourceSoftTracker<RenderDataMetaFile, ColumnRenderSource>
{
public RenderDataSourceSoftRef(RenderDataMetaFile metaFile, ColumnRenderSource data)
{
super(metaFile, data, RENDER_DATA_GARBAGE_COLLECTED_QUEUE);
RENDER_DATA_SOFT_REFS.add(this);
}
@Override
public void close() { RENDER_DATA_SOFT_REFS.remove(this); }
}
/** wrapper for a {@link SoftReference} so we can track and manually remove unused sources */
public static abstract class AbstractDataSourceSoftTracker<TMetaFile extends AbstractMetaDataContainerFile, TDataSource> extends SoftReference<TDataSource> implements Closeable
{
public final TMetaFile metaFile;
public final long createdMsTime;
private long expirationMsTime;
public AbstractDataSourceSoftTracker(TMetaFile metaFile, TDataSource dataSource, ReferenceQueue<TDataSource> referenceQueue)
{
super(dataSource, referenceQueue);
this.metaFile = metaFile;
this.createdMsTime = System.currentTimeMillis();
this.expirationMsTime = System.currentTimeMillis();
}
public void updateLastAccessedTime() { this.expirationMsTime = System.currentTimeMillis() + MS_TO_EXPIRE_DATA_SOURCE; }
public long getExpirationMsTime() { return this.expirationMsTime; }
public boolean isDataSourceExpired() { return this.expirationMsTime > System.currentTimeMillis(); }
@Override
public TDataSource get()
{
this.updateLastAccessedTime();
return super.get();
}
/**
* Gets the underlying datasource without updating the {@link AbstractDataSourceSoftTracker#expirationMsTime}
* Note: this still updates {@link SoftReference}'s timestamp variable which may prevent the JVM from
* marking this reference as valid for deletion.
*/
public TDataSource silentGet() { return super.get(); }
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java
+342 -348
View File
@@ -26,37 +26,55 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIn
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.sql.FullDataRepo;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.channels.ClosedByInterruptException;
import java.nio.channels.ClosedChannelException;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
public class FullDataFileHandler implements IFullDataSourceProvider
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final Timer DELAYED_SAVE_TIMER = new Timer();
/** How long a data source must remain un-modified before being written to disk. */
private static final int SAVE_DELAY_IN_MS = 4_000;
protected final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> loadedMetaFileBySectionPos = new ConcurrentHashMap<>();
protected final ConcurrentHashMap<DhSectionPos, IFullDataSource> unsavedDataSourceBySectionPos = new ConcurrentHashMap<>();
protected final ConcurrentHashMap<DhSectionPos, TimerTask> saveTimerTasksBySectionPos = new ConcurrentHashMap<>();
protected final ReentrantLock[] updateLockArray;
protected final IDhLevel level;
protected final File saveDir;
protected final AtomicInteger topDetailLevelRef = new AtomicInteger(0);
/**
* The highest numerical detail level known about.
* Used when determining which positions to update.
*/
protected final AtomicInteger topSectionDetailLevelRef;
protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET;
public final FullDataRepo fullDataRepo;
@@ -79,6 +97,14 @@ public class FullDataFileHandler implements IFullDataSourceProvider
LOGGER.warn("Unable to create full data folder, file saving may fail.");
}
// the lock array's length is double the number of CPU cores so the number of collisions
// should be relatively low without having too many extra locks
this.updateLockArray = new ReentrantLock[Runtime.getRuntime().availableProcessors() * 2];
for (int i = 0; i < this.updateLockArray.length; i++)
{
this.updateLockArray[i] = new ReentrantLock();
}
try
{
@@ -90,13 +116,17 @@ public class FullDataFileHandler implements IFullDataSourceProvider
// or the database update failed
throw new RuntimeException(e);
}
// determine the top detail level currently in the database
int maxSectionDetailLevel = this.fullDataRepo.getMaxSectionDetailLevel();
this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel);
}
//===============//
// file handling //
//===============//
//==============//
// data reading //
//==============//
/**
* Returns the {@link IFullDataSource} for the given section position. <Br>
@@ -107,251 +137,330 @@ public class FullDataFileHandler implements IFullDataSourceProvider
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
@Override
public CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos)
public CompletableFuture<IFullDataSource> getAsync(DhSectionPos pos)
{
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true);
if (metaFile == null)
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
// future wrapper necessary in order to handle file read errors
CompletableFuture<IFullDataSource> futureWrapper = new CompletableFuture<>();
metaFile.getOrLoadCachedDataSourceAsync().exceptionally((e) ->
{
FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, e);
futureWrapper.completeExceptionally(e);
return null; // return value doesn't matter
})
.whenComplete((dataSource, e) ->
{
futureWrapper.complete(dataSource);
});
return futureWrapper;
return CompletableFuture.supplyAsync(() -> this.get(pos), executor);
}
@Override
public FullDataMetaFile getFileIfExist(DhSectionPos pos) { return this.getLoadOrMakeFile(pos, false); }
protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile)
/**
* Should be used in internal methods where we are already running on a file handler thread.
* @see FullDataFileHandler#getAsync(DhSectionPos)
*/
protected IFullDataSource get(DhSectionPos pos)
{
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos);
if (metaFile != null)
// used the unsaved data source if present
if (this.unsavedDataSourceBySectionPos.containsKey(pos))
{
return metaFile;
return this.unsavedDataSourceBySectionPos.get(pos);
}
// an unsaved data source isn't present
// check the database
// check if the file exists, but hasn't been loaded
MetaDataDto metaDataDto = this.fullDataRepo.getByPrimaryKey(pos.serialize());
if (metaDataDto != null)
{
synchronized (this)
{
// Double check locking for loading file, as loading file means also loading the metadata, which
// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
metaFile = this.loadedMetaFileBySectionPos.get(pos);
if (metaFile != null)
{
return metaFile; // someone else loaded it already.
}
try
{
metaFile = FullDataMetaFile.createFromExistingDto(this, this.level, metaDataDto);
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
this.loadedMetaFileBySectionPos.put(pos, metaFile);
return metaFile;
}
catch (IOException e)
{
LOGGER.error("Failed to read meta data file at pos " + pos + ": ", e);
this.fullDataRepo.delete(metaDataDto);
}
}
}
// increase the top detail level if necessary
this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
if (!allowCreateFile)
{
return null;
}
// File does not exist, create it.
// In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS
// to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects.
IFullDataSource dataSource = null;
try
{
metaFile = FullDataMetaFile.createNewDtoForPos(this, this.level, pos);
}
catch (IOException e)
{
LOGGER.error("IOException on creating new data file at {}", pos, e);
return null;
}
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
// This is a Compare And Swap with expected null value.
FullDataMetaFile metaFileCas = this.loadedMetaFileBySectionPos.putIfAbsent(pos, metaFile);
return metaFileCas == null ? metaFile : metaFileCas;
}
/**
* Populates the preexistingFiles and missingFilePositions ArrayLists.
*
* @param preexistingFiles the list of {@link FullDataMetaFile}'s that have been created for the given position.
* @param missingFilePositions the list of {@link DhSectionPos}'s that don't have {@link FullDataMetaFile} created for them yet.
*/
protected void getDataFilesForPosition(
DhSectionPos effectivePos, DhSectionPos posAreaToGet,
ArrayList<FullDataMetaFile> preexistingFiles, ArrayList<DhSectionPos> missingFilePositions)
{
byte sectionDetail = posAreaToGet.getDetailLevel();
boolean allEmpty = true;
final DhSectionPos.DhMutableSectionPos subPos = new DhSectionPos.DhMutableSectionPos((byte)0, 0, 0);
// get all existing files for this position
outerLoop:
while (--sectionDetail >= this.minDetailLevel)
{
DhLodPos minPos = posAreaToGet.getMinCornerLodPos().getCornerLodPos(sectionDetail);
int count = posAreaToGet.getSectionBBoxPos().getWidthAtDetail(sectionDetail);
for (int xOffset = 0; xOffset < count; xOffset++)
MetaDataDto dto = this.fullDataRepo.getByPrimaryKey(pos.serialize());
if (dto != null)
{
for (int zOffset = 0; zOffset < count; zOffset++)
{
subPos.mutate(sectionDetail, xOffset + minPos.x, zOffset + minPos.z);
LodUtil.assertTrue(posAreaToGet.overlapsExactly(effectivePos) && subPos.overlapsExactly(posAreaToGet));
//TODO: The following check is temporary as we only sample corner points, which means
// on a very different level, we may not need the entire section at all.
if (!CompleteFullDataSource.firstDataPosCanAffectSecond(effectivePos, subPos))
{
continue;
}
// check if a file for this pos is loaded or exists
if (this.loadedMetaFileBySectionPos.containsKey(subPos) || this.fullDataRepo.existsWithPrimaryKey(subPos.serialize()))
{
allEmpty = false;
break outerLoop;
}
}
}
}
if (allEmpty)
{
// there are no children to this quad tree,
// add this leaf's position
missingFilePositions.add(posAreaToGet);
}
else
{
// there are children in this quad tree, search them
this.recursiveGetDataFilesForPosition(0, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions);
this.recursiveGetDataFilesForPosition(1, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions);
this.recursiveGetDataFilesForPosition(2, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions);
this.recursiveGetDataFilesForPosition(3, effectivePos, posAreaToGet, preexistingFiles, missingFilePositions);
}
}
private void recursiveGetDataFilesForPosition(int childIndex, DhSectionPos basePos, DhSectionPos pos, ArrayList<FullDataMetaFile> preexistingFiles, ArrayList<DhSectionPos> missingFilePositions)
{
DhSectionPos childPos = pos.getChildByIndex(childIndex);
if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos))
{
// get or load the file if necessary
if (!this.loadedMetaFileBySectionPos.containsKey(childPos))
{
this.getLoadOrMakeFile(childPos, false);
}
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(childPos);
if (metaFile != null)
{
// we have reached a populated leaf node in the quad tree
preexistingFiles.add(metaFile);
}
else if (childPos.getDetailLevel() == this.minDetailLevel)
{
// we have reached an empty leaf node in the quad tree
missingFilePositions.add(childPos);
// load from file
AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.baseMetaData.dataType, dto.baseMetaData.binaryDataFormatVersion);
dataSource = loader.loadDataSource(dto, this.level);
}
else
{
// recursively traverse down the tree
this.getDataFilesForPosition(basePos, childPos, preexistingFiles, missingFilePositions);
// attempt to create from any existing files
dataSource = this.createNewDataSourceFromExistingDtos(pos);
}
}
}
public void ForEachFile(Consumer<FullDataMetaFile> consumer) { this.loadedMetaFileBySectionPos.values().forEach(consumer); }
//=============//
// data saving //
//=============//
/** This call is concurrent. I.e. it supports being called by multiple threads at the same time. */
@Override
public void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkDataView)
{
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos();
LodUtil.assertTrue(chunkSectionPos.overlapsExactly(sectionPos), "Chunk " + chunkSectionPos + " does not overlap section " + sectionPos);
chunkSectionPos = chunkSectionPos.convertNewToDetailLevel((byte) this.minDetailLevel);
this.writeChunkDataToMetaFile(chunkSectionPos, chunkDataView);
}
private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData)
{
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos);
if (metaFile != null)
catch (InterruptedException ignore) { }
catch (IOException e)
{
// there is a file for this position
metaFile.addToWriteQueue(chunkData);
LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e);
}
if (sectionPos.getDetailLevel() <= this.topDetailLevelRef.get())
return dataSource;
}
/** Creates a new data source using any DTOs already present in the database. */
protected IFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos)
{
IIncompleteFullDataSource newFullDataSource = this.makeEmptyDataSource(pos);
boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get();
if (showFullDataFileSampling)
{
// recursively attempt to get the meta file for this position
this.writeChunkDataToMetaFile(sectionPos.getParentPos(), chunkData);
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA),
0.2, 32f));
}
// TODO replace with a SQL query, it should be much faster
ArrayList<DhSectionPos> samplePosList = new ArrayList<>();
ArrayList<DhSectionPos> possibleChildList = new ArrayList<>();
pos.forEachChild((childPos) ->
{
if (childPos.getDetailLevel() > this.minDetailLevel)
{
possibleChildList.add(childPos);
}
});
while (possibleChildList.size() != 0)
{
DhSectionPos possiblePos = possibleChildList.remove(possibleChildList.size()-1);
if (this.fullDataRepo.existsWithPrimaryKey(possiblePos.serialize()))
{
samplePosList.add(possiblePos);
}
else
{
possiblePos.forEachChild((childPos) ->
{
if (childPos.getDetailLevel() > this.minDetailLevel)
{
possibleChildList.add(childPos);
}
});
}
}
// read in the existing data
for (int i = 0; i < samplePosList.size(); i++)
{
DhSectionPos samplePos = samplePosList.get(i);
IFullDataSource sampleDataSource = this.get(samplePos);
if (sampleDataSource == null)
{
// no file was found, this is unexpected, but can be ignored
continue;
}
if (showFullDataFileSampling)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA.darker()),
0.2, 32f));
}
try
{
newFullDataSource.sampleFrom(sampleDataSource);
}
catch (Exception e)
{
LOGGER.warn("Unable to sample "+sampleDataSource.getSectionPos()+" into "+newFullDataSource.getSectionPos(), e);
}
}
// promotion may happen if all children are fully populated
return newFullDataSource.tryPromotingToCompleteDataSource();
}
/** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */
@Override
public CompletableFuture<Void> flushAndSaveAsync()
{
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
for (FullDataMetaFile metaFile : this.loadedMetaFileBySectionPos.values())
{
futures.add(metaFile.flushAndSaveAsync());
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
//===============//
// data updating //
//===============//
@Override
public CompletableFuture<Void> flushAndSaveAsync(DhSectionPos sectionPos)
public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkDataView)
{
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos);
if (metaFile == null)
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET);
this.recursivelyUpdateDataSourcesAsync(chunkSectionPos, chunkDataView);
}
/** Updates every data source from this position up to {@link FullDataFileHandler#topSectionDetailLevelRef} */
private void recursivelyUpdateDataSourcesAsync(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView)
{
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
return;
}
// update up until we reach the highest available data source
if (pos.getDetailLevel() > this.topSectionDetailLevelRef.get())
{
return;
}
executor.execute(() ->
{
DhSectionPos chunkSectionPos = chunkDataView.getSectionPos();
LodUtil.assertTrue(chunkSectionPos.overlapsExactly(pos), "Update failed, chunk [" + chunkSectionPos + "] does not overlap section [" + pos + "].");
// update this pos
this.updateDataSourceAtPos(pos, chunkDataView);
// recursively update the parent pos
DhSectionPos parentPos = pos.getParentPos();
this.recursivelyUpdateDataSourcesAsync(parentPos, chunkDataView);
});
}
private void updateDataSourceAtPos(DhSectionPos pos, ChunkSizedFullDataAccessor chunkDataView)
{
// a lock is necessary to prevent two threads from writing to the same position at once,
// if that happens only the second update will apply and the LOD will end up with hole(s)
ReentrantLock updateLock = this.getUpdateLockForPos(pos);
try
{
updateLock.lock();
// get or create the data source
IFullDataSource fullDataSource = this.get(pos);
if (fullDataSource == null)
{
fullDataSource = this.makeEmptyDataSource(pos);
}
fullDataSource.update(chunkDataView);
// try promoting the datasource
if (fullDataSource instanceof IIncompleteFullDataSource)
{
IIncompleteFullDataSource incompleteFullDataSource = (IIncompleteFullDataSource) fullDataSource;
fullDataSource = incompleteFullDataSource.tryPromotingToCompleteDataSource();
}
this.queueDelayedSave(fullDataSource);
}
catch (Exception e)
{
LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e);
}
finally
{
updateLock.unlock();
}
}
/**
* Queues the given data source to save after {@link FullDataFileHandler#SAVE_DELAY_IN_MS}
* milliseconds have passed without any additional modifications. <br> <br>
*
* This prevents repeatedly reading/writing the same data source to/from disk if said
* source is currently being updated via world gen or chunk modifications.
* This drastically reduces disk usage and improves performance.
*/
private void queueDelayedSave(IFullDataSource fullDataSource)
{
DhSectionPos pos = fullDataSource.getSectionPos();
// put the data source in memory until it can be flushed to disk
this.unsavedDataSourceBySectionPos.put(pos, fullDataSource);
TimerTask task = new TimerTask()
{
@Override
public void run()
{
try
{
final IFullDataSource finalFullDataSource = FullDataFileHandler.this.unsavedDataSourceBySectionPos.remove(pos);
// this can rarely happen due to imperfect concurrency handling,
// if the data source is null that just means it has already been saved so nothing needs to be done
if (finalFullDataSource != null)
{
FullDataFileHandler.this.writeDataSourceToFile(finalFullDataSource, (bufferedOutputStream) ->
{
try
{
finalFullDataSource.writeToStream(bufferedOutputStream, FullDataFileHandler.this.level);
}
catch (Exception e)
{
// if this try catch isn't included an empty exception will be thrown instead, which makes debugging extremely painful
LOGGER.error("Error writing data stream for pos: [" + finalFullDataSource.getSectionPos() + "], error: " + e.getMessage(), e);
}
});
}
}
catch (ClosedByInterruptException e) // thrown by buffers that are interrupted
{
// expected if the file handler is shut down, the exception can be ignored
//LOGGER.warn("FullData file writing interrupted.", e);
}
catch (IOException e)
{
LOGGER.error("Failed to save updated data for section " + pos, e);
}
}
};
DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS);
//
TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task);
if (oldTask != null)
{
oldTask.cancel();
}
}
private void writeDataSourceToFile(IFullDataSource fullDataSource, AbstractMetaDataContainerFile.IMetaDataWriterFunc<DhDataOutputStream> dataWriterFunc) throws IOException
{
LodUtil.assertTrue(fullDataSource != null);
boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get();
if (showFullDataFileStatus)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW),
0.2, 16f));
}
try
{
// write the outputs to a stream to prep for writing to the database
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// the order of these streams is important, otherwise the checksum won't be calculated
CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32());
// normally a DhStream should be the topmost stream to prevent closing the stream accidentally,
// but since this stream will be closed immediately after writing anyway, it won't be an issue
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut);
dataWriterFunc.writeBinaryDataToStream(compressedOut);
compressedOut.flush();
int checksum = (int) checkedOut.getChecksum().getValue();
byteArrayOutputStream.close();
// save the DTO
BaseMetaData baseMetaData = new BaseMetaData(fullDataSource.getSectionPos(), checksum,
fullDataSource.getDataDetailLevel(), fullDataSource.getWorldGenStep(), fullDataSource.getDataTypeName(),
fullDataSource.getDataFormatVersion());
MetaDataDto newDto = new MetaDataDto(baseMetaData, byteArrayOutputStream.toByteArray());
this.fullDataRepo.save(newDto);
}
catch (ClosedChannelException e) // includes ClosedByInterruptException
{
// expected if the file handler is shut down, the exception can be ignored
//LOGGER.warn(AbstractMetaDataContainerFile.class.getSimpleName()+" file writing interrupted. Error: "+e.getMessage());
}
return metaFile.flushAndSaveAsync();
}
//================//
// helper methods //
//================//
/** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */
protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; }
protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos)
{
@@ -360,136 +469,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider
LowDetailIncompleteFullDataSource.createEmpty(pos);
}
/**
* Populates the given data source using the given array of files
* @param usePooledDataSources if enabled the data sources necessary for this sampling will not be stored beyond what is necessary for the sampling.
* This helps reduce garbage collector pressure if the data sources will never be used again.
*/
protected CompletableFuture<IIncompleteFullDataSource> sampleFromFileArray(IIncompleteFullDataSource recipientFullDataSource, ArrayList<FullDataMetaFile> existingFiles, boolean usePooledDataSources)
{
boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileSampling.get();
if (showFullDataFileSampling)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(recipientFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA),
0.2, 32f));
}
// read in the existing data
final ArrayList<CompletableFuture<IFullDataSource>> sampleDataFutures = new ArrayList<>(existingFiles.size());
for (int i = 0; i < existingFiles.size(); i++)
{
FullDataMetaFile existingFile = existingFiles.get(i);
CompletableFuture<IFullDataSource> loadFileFuture = usePooledDataSources ? existingFile.getDataSourceWithoutCachingAsync() : existingFile.getOrLoadCachedDataSourceAsync();
CompletableFuture<IFullDataSource> sampleSourceFuture = loadFileFuture.whenComplete((existingFullDataSource, ex) ->
{
if (existingFullDataSource == null || ex != null)
{
// Ignore file read errors
//LOGGER.warn(recipientFullDataSource.getSectionPos()+" sample from, file read error for file "+existingFile.pos+": "+ex.getMessage(), ex);
return;
}
// can happen if data source caching isn't working correctly
LodUtil.assertTrue(existingFile.pos.equals(existingFullDataSource.getSectionPos()), "Data source returned the wrong position, pooled data source: ["+usePooledDataSources+"]. Expected: ["+existingFile.pos+"] actual: ["+existingFullDataSource.getSectionPos()+"].");
if (showFullDataFileSampling)
{
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(recipientFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA.darker()),
0.2, 32f));
}
try
{
recipientFullDataSource.sampleFrom(existingFullDataSource);
}
catch (Exception e)
{
LOGGER.warn("Unable to sample "+existingFullDataSource.getSectionPos()+" into "+recipientFullDataSource.getSectionPos(), e);
//throw e;
}
// return the pooled data source if necessary
if (usePooledDataSources)
{
// pooling temporary data sources massively reduces garbage collector overhead when just sampling (going from ~8 GB/sec to ~90 MB/sec)
// get the data loader
AbstractFullDataSourceLoader dataSourceLoader;
if (existingFile.fullDataSourceLoader != null)
{
dataSourceLoader = existingFile.fullDataSourceLoader;
}
else
{
// shouldn't normally happen, but sometimes does
dataSourceLoader = AbstractFullDataSourceLoader.getLoader(existingFile.baseMetaData.dataType, existingFile.baseMetaData.binaryDataFormatVersion);
}
dataSourceLoader.returnPooledDataSource(existingFullDataSource);
}
});
sampleDataFutures.add(sampleSourceFuture);
}
return CompletableFuture.allOf(sampleDataFutures.toArray(new CompletableFuture[0]))
.thenApply(voidObj -> recipientFullDataSource);
}
protected void makeFiles(ArrayList<DhSectionPos> posList, ArrayList<FullDataMetaFile> output)
{
for (DhSectionPos missingPos : posList)
{
FullDataMetaFile newFile = this.getLoadOrMakeFile(missingPos, true);
if (newFile != null)
{
output.add(newFile);
}
}
}
@Override
public CompletableFuture<IFullDataSource> onDataFileCreatedAsync(FullDataMetaFile file)
{
DhSectionPos pos = file.pos;
IIncompleteFullDataSource source = this.makeEmptyDataSource(pos);
ArrayList<FullDataMetaFile> existFiles = new ArrayList<>();
ArrayList<DhSectionPos> missing = new ArrayList<>();
this.getDataFilesForPosition(pos, pos, existFiles, missing);
LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty());
if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos))
{
// None exist.
return CompletableFuture.completedFuture(source);
}
else
{
this.makeFiles(missing, existFiles);
return this.sampleFromFileArray(source, existFiles, true).thenApply(IIncompleteFullDataSource::tryPromotingToCompleteDataSource)
.exceptionally((e) ->
{
FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, e);
return null;
});
}
}
protected FullDataMetaFile removeCorruptedFile(DhSectionPos pos, Throwable exception)
{
LOGGER.error("Error reading Data file [" + pos + "]", exception);
this.fullDataRepo.deleteByPrimaryKey(pos.serialize());
// remove the FullDataMetaFile since the old one was corrupted
this.loadedMetaFileBySectionPos.remove(pos);
// create a new FullDataMetaFile to write new data to
return this.getLoadOrMakeFile(pos, true);
}
//=========//
@@ -497,9 +476,24 @@ public class FullDataFileHandler implements IFullDataSourceProvider
//=========//
@Override
public void close() {
FullDataMetaFile.checkAndLogPhantomDataSourceLifeCycles();
this.fullDataRepo.close();
public void close()
{
LOGGER.info("Closing file handler for level: ["+this.level+"], saving ["+this.saveTimerTasksBySectionPos.size()+"] positions.");
Enumeration<DhSectionPos> list = this.saveTimerTasksBySectionPos.keys();
while (list.hasMoreElements())
{
DhSectionPos pos = list.nextElement();
TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos);
if (saveTask != null)
{
saveTask.run();
saveTask.cancel();
}
}
LOGGER.info("File handler saving complete, closing repo.");
this.fullDataRepo.close();
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java
-787
View File
@@ -1,787 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.fullDatafile;
import java.awt.*;
import java.io.*;
import java.lang.ref.*;
import java.nio.channels.ClosedByInterruptException;
import java.util.Set;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
import com.seibel.distanthorizons.core.file.DataSourceReferenceTracker;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.AtomicsUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.util.threading.ThreadPools;
import org.apache.logging.log4j.Logger;
/** Represents a File that contains a {@link IFullDataSource}. */
public class FullDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName());
// === Object lifetime tracking ===
/** if true both data source creation and garbage collection will be logged */
private static final boolean LOG_DATA_SOURCE_LIVES = false;
private static final ReferenceQueue<IFullDataSource> LIFE_CYCLE_DEBUG_QUEUE = new ReferenceQueue<>();
private static final ReferenceQueue<IFullDataSource> SOFT_REF_DEBUG_QUEUE = new ReferenceQueue<>();
private static final Set<DataObjTracker> LIFE_CYCLE_DEBUG_SET = ConcurrentHashMap.newKeySet();
private static final Set<DataObjSoftTracker> SOFT_REF_DEBUG_SET = ConcurrentHashMap.newKeySet();
// ===========================
public boolean doesDtoExist;
/** indicates if this file has been checked for missing sections to generate or not */
public boolean genQueueChecked = false;
public AbstractFullDataSourceLoader fullDataSourceLoader;
public Class<? extends IFullDataSource> fullDataSourceClass;
private volatile boolean needsUpdate = false;
private final IDhLevel level;
private final IFullDataSourceProvider fullDataSourceProvider;
/**
* Can be cleared if the garbage collector determines there isn't enough space. <br><br>
*
* When clearing, don't set to null, instead create a SoftReference containing null.
* This makes null checks simpler.
*/
private DataSourceReferenceTracker.FullDataSourceSoftRef cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this,null);
// two different load futures are used to
// prevent accidentally returning a pooled (non-cached) data source
private final AtomicReference<CompletableFuture<IFullDataSource>> cachedDataSourceLoadFutureRef = new AtomicReference<>(null);
private final AtomicReference<CompletableFuture<IFullDataSource>> pooledDataSourceLoadFutureRef = new AtomicReference<>(null);
// === Concurrent Write tracking ===
private final AtomicReference<GuardedMultiAppendQueue> writeQueueRef = new AtomicReference<>(new GuardedMultiAppendQueue());
private GuardedMultiAppendQueue backWriteQueue = new GuardedMultiAppendQueue();
// ===========================
//==============//
// constructors //
//==============//
/**
* NOTE: should only be used if there is NOT an existing file.
* @throws IOException if a file already exists for this position
*/
public static FullDataMetaFile createNewDtoForPos(IFullDataSourceProvider fullDataSourceProvider, IDhLevel clientLevel, DhSectionPos pos) throws IOException { return new FullDataMetaFile(fullDataSourceProvider, clientLevel, pos); }
private FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, DhSectionPos pos) throws IOException
{
super(pos);
checkAndLogPhantomDataSourceLifeCycles();
this.fullDataSourceProvider = fullDataSourceProvider;
this.level = level;
LodUtil.assertTrue(this.baseMetaData == null);
this.doesDtoExist = false;
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus);
}
/**
* NOTE: should only be used if there IS an existing file.
* @throws IOException if the file was formatted incorrectly
* @throws FileNotFoundException if no file exists for the given path
*/
public static FullDataMetaFile createFromExistingDto(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, MetaDataDto metaDataDto) throws IOException { return new FullDataMetaFile(fullDataSourceProvider, level, metaDataDto); }
private FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, MetaDataDto metaDataDto) throws IOException
{
super(metaDataDto.baseMetaData);
checkAndLogPhantomDataSourceLifeCycles();
this.fullDataSourceProvider = fullDataSourceProvider;
this.level = level;
LodUtil.assertTrue(this.baseMetaData != null);
this.doesDtoExist = true;
this.fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(this.baseMetaData.dataType, this.baseMetaData.binaryDataFormatVersion);
if (this.fullDataSourceLoader == null)
{
throw new IOException("Invalid file: Data type loader not found: " + this.baseMetaData.dataType + "(v" + this.baseMetaData.binaryDataFormatVersion + ")");
}
this.fullDataSourceClass = this.fullDataSourceLoader.fullDataSourceClass;
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus);
}
//==========//
// get data //
//==========//
/**
* Try get cached data source. Used for temp impl of re-queueing world gen tasks.
* (Read-only access! As writes should always be done async)
*/
public IFullDataSource getCachedDataSourceNowOrNull()
{
checkAndLogPhantomDataSourceLifeCycles();
return this.cachedFullDataSourceRef.get();
}
/** @return if any data was cleared */
public boolean clearCachedDataSource()
{
boolean dataExists = this.cachedFullDataSourceRef.get() != null;
if (dataExists)
{
this.cachedFullDataSourceRef.close();
this.cachedFullDataSourceRef.clear();
}
return dataExists;
}
public CompletableFuture<IFullDataSource> getDataSourceWithoutCachingAsync() { return this.getOrLoadCachedDataSourceAsync(true); }
public CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync() { return this.getOrLoadCachedDataSourceAsync(true); }
/**
* Synchronized to help prevent issues where multiple threads try to read as cached and un-cached at the same time.
* Hopefully isn't necessary and could potentially be removed in the future.
*/
private synchronized CompletableFuture<IFullDataSource> getOrLoadCachedDataSourceAsync(boolean cacheLoadingSource)
{
checkAndLogPhantomDataSourceLifeCycles();
AtomicReference<CompletableFuture<IFullDataSource>> dataSourceLoadFutureRef = cacheLoadingSource ? this.cachedDataSourceLoadFutureRef : this.pooledDataSourceLoadFutureRef;
//========================//
// use the pre-existing //
// load future if present //
//========================//
CompletableFuture<IFullDataSource> preExistingLoadFuture = dataSourceLoadFutureRef.get();
if (preExistingLoadFuture != null)
{
return preExistingLoadFuture;
}
//========================//
// attempt to get the //
// cached data if present //
//========================//
CompletableFuture<IFullDataSource> potentialLoadFuture = null;
if (cacheLoadingSource)
{
potentialLoadFuture = this.getCachedDataSourceAndUpdateIfNeededAsync();
}
if (potentialLoadFuture != null)
{
// return the in-process future
return potentialLoadFuture;
}
else
{
// there is no cached data, we'll have to load it
// create a new load future if necessary
potentialLoadFuture = new CompletableFuture<>();
if (!dataSourceLoadFutureRef.compareAndSet(null, potentialLoadFuture))
{
// two threads attempted to start this job at the same time, only use the first future
// (shouldn't happen since this method is synchronized, but just in case)
potentialLoadFuture = dataSourceLoadFutureRef.get();
}
}
final CompletableFuture<IFullDataSource> dataSourceLoadFuture = potentialLoadFuture;
if (!this.doesDtoExist)
{
//==================//
// create a new DTO //
// and data source //
//==================//
this.fullDataSourceProvider.onDataFileCreatedAsync(this)
.thenApply((fullDataSource) ->
{
AbstractFullDataSourceLoader dataSourceLoader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getBinaryDataFormatVersion());
this.baseMetaData = new BaseMetaData(
fullDataSource.getSectionPos(), -1,
fullDataSource.getDataDetailLevel(), fullDataSource.getWorldGenStep(),
(dataSourceLoader == null ? null : dataSourceLoader.datatype), fullDataSource.getBinaryDataFormatVersion(), Long.MAX_VALUE);
return fullDataSource;
})
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource))
.thenAccept((fullDataSource) ->
{
dataSourceLoadFuture.complete(fullDataSource);
dataSourceLoadFutureRef.set(null);
});
}
else
{
//=========================//
// load the data from file //
//=========================//
if (this.baseMetaData == null)
{
throw new IllegalStateException("Meta data not loaded!");
}
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor != null && !executor.isTerminated())
{
// load the data source
CompletableFuture.supplyAsync(() ->
{
// Load the file.
IFullDataSource fullDataSource;
try (InputStream inputStream = this.getInputStream();
DhDataInputStream compressedStream = new DhDataInputStream(inputStream))
{
if (cacheLoadingSource)
{
fullDataSource = this.fullDataSourceLoader.loadDataSource(this, compressedStream, this.level);
}
else
{
fullDataSource = this.fullDataSourceLoader.loadTemporaryDataSource(this, compressedStream, this.level);
}
}
catch (Exception ex)
{
LOGGER.error("Full Data Load error for pos ["+this.pos+"], error: "+ ex.getMessage(), ex);
dataSourceLoadFuture.completeExceptionally(ex);
dataSourceLoadFutureRef.set(null);
// can happen if there is a missing file or the file was incorrectly formatted, or terminated early
throw new CompletionException(ex);
}
return fullDataSource;
}, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, cacheLoadingSource))
.thenAccept((fullDataSource) ->
{
dataSourceLoadFuture.complete(fullDataSource);
dataSourceLoadFutureRef.set(null);
});
}
else
{
// don't load anything if the provider has been shut down
dataSourceLoadFuture.complete(null);
dataSourceLoadFutureRef.set(null);
return dataSourceLoadFuture;
}
}
return dataSourceLoadFuture;
}
/** @return returns null if {@link FullDataMetaFile#cachedFullDataSourceRef} is empty and no cached {@link IFullDataSource} exists. */
private CompletableFuture<IFullDataSource> getCachedDataSourceAndUpdateIfNeededAsync()
{
// attempt to get the cached data source
IFullDataSource cachedFullDataSource = this.cachedFullDataSourceRef.get();
if (cachedFullDataSource == null)
{
// no cached data exists and no one is trying to load it
return null;
}
else
{
// cached data exists
boolean dataNeedsUpdating = !this.writeQueueRef.get().queue.isEmpty() || this.needsUpdate;
if (!dataNeedsUpdating)
{
// return the cached data
return CompletableFuture.completedFuture(cachedFullDataSource);
}
else
{
// update the data using the write queue, wait for the update to finish, then return the data source
// Create a new future if one doesn't already exist
CompletableFuture<IFullDataSource> newFuture = new CompletableFuture<>();
CompletableFuture<IFullDataSource> oldFuture = AtomicsUtil.compareAndExchange(this.cachedDataSourceLoadFutureRef, null, newFuture);
if (oldFuture != null)
{
// An update is already in progress, return its future.
return oldFuture;
}
else
{
ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor();
if (executor != null && !executor.isTerminated())
{
// wait for the update to finish before returning the data source
CompletableFuture.supplyAsync(() -> cachedFullDataSource, executor)
.thenCompose((fullDataSource) -> this.applyWriteQueueAndSaveAsync(fullDataSource, true))
.thenAccept((fullDataSource) ->
{
newFuture.complete(fullDataSource);
this.cachedDataSourceLoadFutureRef.set(null);
});
}
else
{
// don't update anything if the provider has been shut down
this.cachedDataSourceLoadFutureRef.set(null);
newFuture.complete(null);
}
return newFuture;
}
}
}
}
//===============//
// data updating //
//===============//
/**
* Adds the given {@link ChunkSizedFullDataAccessor} to the write queue,
* which will be applied to the object at some undefined time in the future.
*/
public void addToWriteQueue(ChunkSizedFullDataAccessor chunkAccessor)
{
checkAndLogPhantomDataSourceLifeCycles();
DhLodPos chunkLodPos = new DhLodPos(LodUtil.CHUNK_DETAIL_LEVEL, chunkAccessor.chunkPos.x, chunkAccessor.chunkPos.z);
LodUtil.assertTrue(this.pos.getSectionBBoxPos().overlapsExactly(chunkLodPos), "Chunk pos " + chunkLodPos + " doesn't exactly overlap with section " + this.pos);
//LOGGER.info("Write Chunk {} to file {}", chunkPos, pos);
GuardedMultiAppendQueue writeQueue = this.writeQueueRef.get();
// Using read lock is OK, because the queue's underlying data structure is thread-safe.
// This lock is only used to insure on polling the queue, that the queue is not being
// modified by another thread.
ReentrantReadWriteLock.ReadLock appendLock = writeQueue.appendLock.readLock();
appendLock.lock();
try
{
writeQueue.queue.add(chunkAccessor);
}
finally
{
appendLock.unlock();
}
this.flushAndSaveAsync();
//LOGGER.info("write queue length for pos "+this.pos+": " + writeQueue.queue.size());
}
/** Applies any queued {@link ChunkSizedFullDataAccessor} to this metadata's {@link IFullDataSource} and writes the data to file. */
public CompletableFuture<Void> flushAndSaveAsync()
{
checkAndLogPhantomDataSourceLifeCycles();
boolean isEmpty = this.writeQueueRef.get().queue.isEmpty() && !this.needsUpdate;
if (!isEmpty)
{
// This will flush the data to disk.
return this.getDataSourceWithoutCachingAsync().thenApply((fullDataSource) -> null /* ignore the result, just wait for the load to finish*/ );
}
else
{
return CompletableFuture.completedFuture(null);
}
}
public void markNeedsUpdate() { this.needsUpdate = true; }
//===========//
// debugging //
//===========//
/** can be used to log when data sources have been garbage collected */
public static void checkAndLogPhantomDataSourceLifeCycles()
{
DataObjTracker phantomRef = (DataObjTracker) LIFE_CYCLE_DEBUG_QUEUE.poll();
// wait for the tracker to be garbage collected
while (phantomRef != null)
{
if (LOG_DATA_SOURCE_LIVES)
{
LOGGER.info("Full Data at pos: " + phantomRef.pos + " has been freed. [" + LIFE_CYCLE_DEBUG_SET.size() + "] Full Data sources remaining.");
}
phantomRef.close();
phantomRef = (DataObjTracker) LIFE_CYCLE_DEBUG_QUEUE.poll();
}
DataObjSoftTracker softRef = (DataObjSoftTracker) SOFT_REF_DEBUG_QUEUE.poll();
while (softRef != null)
{
if (LOG_DATA_SOURCE_LIVES)
{
LOGGER.info("Full Data at pos: " + softRef.file.pos + " has been soft released.");
}
softRef.close();
softRef = (DataObjSoftTracker) SOFT_REF_DEBUG_QUEUE.poll();
}
}
@Override
public void debugRender(DebugRenderer debugRenderer)
{
if (this.pos.getDetailLevel() > DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL)
{
return;
}
if (this.needsUpdate)
{
debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 80f, 96f, 0.05f, Color.red));
}
IFullDataSource cachedDataSource = this.cachedFullDataSourceRef.get();
boolean needsUpdate = !this.writeQueueRef.get().queue.isEmpty() || this.needsUpdate;
// determine the color
Color color = Color.black;
if (cachedDataSource != null)
{
if (cachedDataSource instanceof CompleteFullDataSource)
{
color = Color.GREEN;
}
else
{
color = Color.YELLOW;
}
}
else if (this.cachedDataSourceLoadFutureRef.get() != null)
{
color = Color.BLUE;
}
else if (this.doesDtoExist)
{
color = Color.RED;
}
else if (needsUpdate)
{
color = color.darker().darker();
}
debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 80f, 96f, 0.05f, color));
}
//================//
// helper methods //
//================//
// TODO merge with RenderDataMetaFile
/** @return a stream for the data contained in this file, skips the metadata from {@link AbstractMetaDataContainerFile}. */
private InputStream getInputStream() throws IOException
{
MetaDataDto dto = this.fullDataSourceProvider.getRepo().getByPrimaryKey(this.pos.serialize());
return new ByteArrayInputStream(dto.dataArray);
}
/**
* Applies the {@link FullDataMetaFile#writeQueueRef} to the current {@link IFullDataSource}
* and stores the result in {@link FullDataMetaFile#cachedFullDataSourceRef}.
*/
@SuppressWarnings("resource") // due to DataObjTracker and DataObjSoftTracker being created outside a try-catch block
private CompletableFuture<IFullDataSource> applyWriteQueueAndSaveAsync(IFullDataSource fullDataSourceToUpdate, boolean cacheLoadingSource)
{
CompletableFuture<IFullDataSource> completionFuture = new CompletableFuture<>();
boolean dataChanged = this.applyWriteQueueToFullDataSource(fullDataSourceToUpdate);
this.needsUpdate = false;
// attempt to promote the data source
if (fullDataSourceToUpdate instanceof IIncompleteFullDataSource)
{
IFullDataSource newSource = ((IIncompleteFullDataSource) fullDataSourceToUpdate).tryPromotingToCompleteDataSource();
dataChanged |= (newSource != fullDataSourceToUpdate);
fullDataSourceToUpdate = newSource;
}
// the provider may need to modify other files based on this data source changing
this.fullDataSourceProvider.onDataFileUpdateAsync(fullDataSourceToUpdate, this, dataChanged)
.whenComplete((dataFileUpdateResult, ex) ->
{
if (ex != null && !LodUtil.isInterruptOrReject(ex))
{
LOGGER.error("Error updating full meta file ["+this.pos+"]: ", ex);
}
IFullDataSource fullDataSource = dataFileUpdateResult.fullDataSource;
boolean dataSourceChanged = dataFileUpdateResult.dataSourceChanged;
// only save to file if something was changed
if (dataSourceChanged)
{
this.writeDataSource(fullDataSource);
}
// keep track of non-null data sources
if (fullDataSource != null)
{
new DataObjTracker(fullDataSource);
new DataObjSoftTracker(this, fullDataSource);
}
boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get();
boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileSampling.get();
if (showFullDataFileStatus || showFullDataFileSampling)
{
Color color = dataSourceChanged ? Color.GREEN : Color.GREEN.darker().darker();
DebugRenderer.makeParticle(new DebugRenderer.BoxParticle(
new DebugRenderer.Box(this.pos, 64f, 72f, 0.03f, color),
0.2, 32f));
}
if (cacheLoadingSource)
{
if (fullDataSource != null)
{
LodUtil.assertTrue(this.pos.equals(fullDataSource.getSectionPos()), "Attempting to cache a datasource with the wrong position. Meta file pos: [" + this.pos + "], data source pos: [" + fullDataSource.getSectionPos() + "].");
}
// save the updated data source
this.cachedFullDataSourceRef = new DataSourceReferenceTracker.FullDataSourceSoftRef(this, fullDataSource);
}
// the task is complete
completionFuture.complete(fullDataSource);
if (this.needsUpdate)
{
// another update was requested while this update was being processed
if (cacheLoadingSource)
{
this.getOrLoadCachedDataSourceAsync();
}
else
{
this.getDataSourceWithoutCachingAsync();
}
}
});
return completionFuture;
}
/** @return true if the queue was not empty and chunk data was applied to this meta file's {@link IFullDataSource}. */
private boolean applyWriteQueueToFullDataSource(IFullDataSource fullDataSource)
{
// swap the write queue if it has queued chunks.
// Must be done in this order to ensure IWorldGenTaskTracker.isMemoryAddressValid() work properly. See IWorldGenTaskTracker.isMemoryAddressValid() for details.
boolean queueIsEmpty = this.writeQueueRef.get().queue.isEmpty();
if (!queueIsEmpty)
{
this.swapWriteQueues();
for (ChunkSizedFullDataAccessor chunk : this.backWriteQueue.queue)
{
fullDataSource.update(chunk);
}
this.backWriteQueue.queue.clear();
//LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count);
}
return !queueIsEmpty || !this.doesDtoExist;
}
private void swapWriteQueues()
{
GuardedMultiAppendQueue writeQueue = this.writeQueueRef.getAndSet(this.backWriteQueue);
// Acquire write lock and then release it again as we only need to ensure that the queue
// is not being appended to by another thread. Note that the above atomic swap &
// the guarantee that all append first acquire the appendLock means after the locK() call,
// there will be no other threads able to or is currently appending to the queue.
// Note: The above needs the getAndSet() to have at least Release Memory order.
// (not that java supports anything non volatile for getAndSet()...)
writeQueue.appendLock.writeLock().lock();
writeQueue.appendLock.writeLock().unlock();
this.backWriteQueue = writeQueue;
}
private void writeDataSource(IFullDataSource fullDataSource)
{
if (fullDataSource.isEmpty())
{
// delete the empty data source
MetaDataDto dto = this.fullDataSourceProvider.getRepo().getByPrimaryKey(this.pos.serialize());
if (dto != null)
{
this.fullDataSourceProvider.getRepo().delete(dto);
}
this.doesDtoExist = false;
}
else
{
// update the data source and write the new data to file
//LOGGER.info("Saving data file of {}", data.getSectionPos());
try
{
// Write/Update data
LodUtil.assertTrue(this.baseMetaData != null);
// confirm the meta data properties are up to date //
this.baseMetaData.dataDetailLevel = fullDataSource.getDataDetailLevel();
this.fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getBinaryDataFormatVersion());
LodUtil.assertTrue(this.fullDataSourceLoader != null, "No loader for " + fullDataSource.getClass() + " (v" + fullDataSource.getBinaryDataFormatVersion() + ")");
this.fullDataSourceClass = fullDataSource.getClass();
this.baseMetaData.dataType = (this.fullDataSourceLoader == null) ? null : this.fullDataSourceLoader.datatype;
this.baseMetaData.binaryDataFormatVersion = fullDataSource.getBinaryDataFormatVersion();
// save the data to the database //
super.writeToDatabase((bufferedOutputStream) -> fullDataSource.writeToStream((bufferedOutputStream), this.level), this.fullDataSourceProvider.getRepo());
this.doesDtoExist = true;
}
catch (ClosedByInterruptException e) // thrown by buffers that are interrupted
{
// expected if the file handler is shut down, the exception can be ignored
//LOGGER.warn("FullData file writing interrupted.", e);
}
catch (IOException e)
{
LOGGER.error("Failed to save updated data for section " + this.pos, e);
}
}
}
//================//
// helper classes //
//================//
//TODO: use ConcurrentAppendSingleSwapContainer<LodDataSource> instead of below:
private static class GuardedMultiAppendQueue
{
ReentrantReadWriteLock appendLock = new ReentrantReadWriteLock();
ConcurrentLinkedQueue<ChunkSizedFullDataAccessor> queue = new ConcurrentLinkedQueue<>();
}
/** used to debug data source soft reference garbage collection */
private static class DataObjTracker extends PhantomReference<IFullDataSource> implements Closeable
{
public final DhSectionPos pos;
DataObjTracker(IFullDataSource data)
{
super(data, LIFE_CYCLE_DEBUG_QUEUE);
if (LOG_DATA_SOURCE_LIVES)
{
//LOGGER.info("Phantom created on "+data.getSectionPos()+"! count: "+LIFE_CYCLE_DEBUG_SET.size());
}
LIFE_CYCLE_DEBUG_SET.add(this);
this.pos = data.getSectionPos();
}
@Override
public void close() { LIFE_CYCLE_DEBUG_SET.remove(this); }
}
/** used to debug data source soft reference garbage collection */
private static class DataObjSoftTracker extends SoftReference<IFullDataSource> implements Closeable
{
public final FullDataMetaFile file;
DataObjSoftTracker(FullDataMetaFile file, IFullDataSource data)
{
super(data, SOFT_REF_DEBUG_QUEUE);
SOFT_REF_DEBUG_SET.add(this);
this.file = file;
}
@Override
public void close() { SOFT_REF_DEBUG_SET.remove(this); }
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java
+24 -184
View File
@@ -20,9 +20,7 @@
package com.seibel.distanthorizons.core.file.fullDatafile;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.generation.MissingWorldGenPositionFinder;
import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue;
@@ -35,7 +33,6 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import javax.annotation.Nullable;
import java.lang.ref.WeakReference;
import java.util.*;
import java.util.concurrent.*;
@@ -46,7 +43,6 @@ import java.util.function.Function;
public class GeneratedFullDataFileHandler extends FullDataFileHandler
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final Timer CHUNK_GEN_FINISHED_TIMER = new Timer();
private final AtomicReference<IWorldGenerationQueue> worldGenQueueRef = new AtomicReference<>(null);
@@ -55,6 +51,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
/** Used to prevent data sources from being garbage collected before their world gen finishes. */
private final ConcurrentHashMap<DhSectionPos, IFullDataSource> generatingDataSourceByPos = new ConcurrentHashMap<>();
//=============//
// constructor //
//=============//
public GeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); }
@@ -64,37 +66,18 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
//===========//
@Override
public CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos)
{
CompletableFuture<IFullDataSource> future = super.readAsync(pos);
return future.thenApply((dataSource) ->
{
// add world gen tasks for missing columns in the data source
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos);
if (worldGenQueue != null && metaFile != null)
{
this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource);
}
return dataSource;
});
}
@Override
public void onRenderDataFileLoaded(DhSectionPos pos)
protected IFullDataSource get(DhSectionPos pos)
{
IFullDataSource dataSource = super.get(pos);
// add world gen tasks for missing columns in the data source
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, false);
if (worldGenQueue != null && metaFile != null)
if (worldGenQueue != null)
{
metaFile.getDataSourceWithoutCachingAsync().thenApply((fullDataSource) ->
{
this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, fullDataSource);
return fullDataSource;
});
this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, pos, dataSource);
}
return dataSource;
}
@@ -111,27 +94,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
{
boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue);
LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!");
LOGGER.info("Set world gen queue for level "+this.level+" to start.");
this.ForEachFile(metaFile ->
{
IFullDataSource dataSource = metaFile.getCachedDataSourceNowOrNull();
if (dataSource == null)
{
return;
}
metaFile.genQueueChecked = false; // allow the system to check for missing positions again
this.queueWorldGenForMissingColumnsInDataSource(this.worldGenQueueRef.get(), metaFile, dataSource);
if (dataSource instanceof CompleteFullDataSource)
{
return;
}
metaFile.markNeedsUpdate();
});
this.flushAndSaveAsync(); // Trigger an update to the meta files
LOGGER.info("Set world gen queue for level ["+this.level+"].");
}
public void clearGenerationQueue()
@@ -140,14 +103,13 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
this.generatingDataSourceByPos.clear(); // clear the incomplete data sources
}
// TODO what is this here for?
/** Can be used to remove positions that are outside the player's render distance. */
public void removeGenRequestIf(Function<DhSectionPos, Boolean> removeIf)
{
this.generatingDataSourceByPos.forEach((pos, dataSource) ->
{
if (removeIf.apply(pos))
{
//this.worldGenQueueRef.get().cancelGenTasks(pos); // shouldn't this be called if we actually want to stop world gen
this.generatingDataSourceByPos.remove(pos);
}
});
@@ -160,104 +122,14 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
//=================//
public void addWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.add(listener); }
public void removeWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.remove(listener); }
private IFullDataSource tryPromoteDataSource(IIncompleteFullDataSource source)
{
IFullDataSource newSource = source.tryPromotingToCompleteDataSource();
if (newSource instanceof CompleteFullDataSource)
{
this.generatingDataSourceByPos.remove(source.getSectionPos());
}
return newSource;
}
//========//
// events //
//========//
// Try update the gen queue on this data source. If null, then nothing was done.
@Nullable
private CompletableFuture<IFullDataSource> updateFromExistingDataSourcesAsync(FullDataMetaFile file, IIncompleteFullDataSource data, boolean usePooledDataSources)
{
DhSectionPos pos = file.pos;
ArrayList<FullDataMetaFile> existingFiles = new ArrayList<>();
ArrayList<DhSectionPos> missingPositions = new ArrayList<>();
this.getDataFilesForPosition(pos, pos, existingFiles, missingPositions);
if (missingPositions.size() == 1)
{
// Only missing myself. I.e. no child file data exists yet.
return this.tryStartGenTask(file, data);
}
else
{
// There are other data source files to sample from.
this.makeFiles(missingPositions, existingFiles);
return this.sampleFromFileArray(data, existingFiles, usePooledDataSources)
.thenApply(this::tryPromoteDataSource)
.exceptionally((e) ->
{
this.removeCorruptedFile(pos, e);
return null;
});
}
}
@Nullable
private CompletableFuture<IFullDataSource> tryStartGenTask(FullDataMetaFile metaFile, IIncompleteFullDataSource dataSource) // TODO after generation is finished, save and free any full datasources that aren't in use (IE high detail ones below the top)
{
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
if (worldGenQueue != null)
{
this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, metaFile, dataSource);
return CompletableFuture.completedFuture(dataSource);
}
return null;
}
@Override
public CompletableFuture<IFullDataSource> onDataFileCreatedAsync(FullDataMetaFile file)
{
DhSectionPos pos = file.pos;
IIncompleteFullDataSource data = this.makeEmptyDataSource(pos);
CompletableFuture<IFullDataSource> future = this.updateFromExistingDataSourcesAsync(file, data, true);
// Cant start gen task, so return the data
return future == null ? CompletableFuture.completedFuture(data) : future;
}
@Override
public CompletableFuture<DataFileUpdateResult> onDataFileUpdateAsync(IFullDataSource fullDataSource, FullDataMetaFile file, boolean dataChanged)
{
LodUtil.assertTrue(this.fullDataRepo.existsWithPrimaryKey(file.pos.serialize()) || dataChanged);
if (fullDataSource instanceof CompleteFullDataSource)
{
this.generatingDataSourceByPos.remove(fullDataSource.getSectionPos());
}
this.fireOnGenPosSuccessListeners(fullDataSource.getSectionPos());
if (fullDataSource instanceof IIncompleteFullDataSource && !file.genQueueChecked)
{
IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get();
if (worldGenQueue != null)
{
CompletableFuture<IFullDataSource> future = this.updateFromExistingDataSourcesAsync(file, (IIncompleteFullDataSource) fullDataSource, false);
if (future != null)
{
final boolean finalDataChanged = dataChanged;
return future.thenApply((newSource) -> new DataFileUpdateResult(newSource, finalDataChanged));
}
}
}
return CompletableFuture.completedFuture(new DataFileUpdateResult(fullDataSource, dataChanged));
}
private void onWorldGenTaskComplete(WorldGenResult genTaskResult, Throwable exception, GenTask genTask, DhSectionPos pos)
{
if (exception != null)
@@ -270,18 +142,6 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
}
else if (genTaskResult.success)
{
// generation completed, update the files and listener(s)
this.flushAndSaveAsync(pos).join();
// FIXME this is a bad fix to prevent full data sources saving incomplete, causing holes in the world after generation.
// The problem appears to be that the save may be happening too quickly,
// potentially happening before the meta file has the newly generated data added to it.
CHUNK_GEN_FINISHED_TIMER.schedule(new TimerTask()
{
@Override
public void run() { GeneratedFullDataFileHandler.this.flushAndSaveAsync(pos).join(); }
}, 4000L);
this.fireOnGenPosSuccessListeners(pos);
return;
}
@@ -316,24 +176,9 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
// helper methods //
//================//
private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, FullDataMetaFile metaFile, IFullDataSource dataSource)
private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, DhSectionPos pos, IFullDataSource dataSource)
{
// Due to a bug in the current system, some Complete data sources aren't actually complete
// and will need additional generation to finish
//if (dataSource instanceof CompleteFullDataSource)
//{
// return;
//}
if (metaFile.genQueueChecked)
{
// world gen has already been checked for this file
return;
}
metaFile.genQueueChecked = true;
// get the ungenerated pos list
// get the un-generated pos list
byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
ArrayList<DhSectionPos> genPosList = MissingWorldGenPositionFinder.getUngeneratedPosList(dataSource, minGeneratorSectionDetailLevel, true);
@@ -341,17 +186,13 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
ArrayList<CompletableFuture<WorldGenResult>> taskFutureList = new ArrayList<>();
for (DhSectionPos genPos : genPosList)
{
// make sure each meta file has been created (not doing this will prevent down sampling and/or saving the generated data source)
this.getLoadOrMakeFile(genPos, true);
this.getLoadOrMakeFile(metaFile.pos, true);
// queue each gen task
GenTask genTask = new GenTask(dataSource.getSectionPos(), new WeakReference<>(dataSource));
CompletableFuture<WorldGenResult> worldGenFuture = worldGenQueue.submitGenTask(genPos, dataSource.getDataDetailLevel(), genTask);
worldGenFuture.whenComplete((genTaskResult, ex) ->
{
this.onWorldGenTaskComplete(genTaskResult, ex, genTask, genPos);
this.onWorldGenTaskComplete(genTaskResult, ex, genTask, metaFile.pos);
this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos);
});
taskFutureList.add(worldGenFuture);
@@ -361,14 +202,13 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
// mark the data source as generating if necessary
if (taskFutureList.size() != 0)
{
this.generatingDataSourceByPos.put(metaFile.pos, dataSource);
this.generatingDataSourceByPos.put(pos, dataSource);
CompletableFuture.allOf(taskFutureList.toArray(new CompletableFuture[0]))
.whenComplete((voidObj, ex) ->
{
this.generatingDataSourceByPos.remove(pos);
});
}
CompletableFuture.allOf(taskFutureList.toArray(new CompletableFuture[0]))
.whenComplete((voidObj, ex) ->
{
metaFile.flushAndSaveAsync();
this.generatingDataSourceByPos.remove(metaFile.pos);
});
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java
+6 -38
View File
@@ -23,51 +23,19 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedF
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.FullDataRepo;
import org.jetbrains.annotations.Nullable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
/**
* Handles reading, writing, and updating {@link IFullDataSource}'s. <br>
* Should be backed by a database handled by a {@link FullDataRepo}.
*/
public interface IFullDataSourceProvider extends AutoCloseable
{
CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos);
void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData);
CompletableFuture<Void> flushAndSaveAsync();
CompletableFuture<Void> flushAndSaveAsync(DhSectionPos sectionPos);
CompletableFuture<IFullDataSource> getAsync(DhSectionPos pos);
//long getCacheVersion(DhSectionPos sectionPos);
//boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion);
CompletableFuture<IFullDataSource> onDataFileCreatedAsync(FullDataMetaFile file);
default CompletableFuture<DataFileUpdateResult> onDataFileUpdateAsync(IFullDataSource fullDataSource, FullDataMetaFile file, boolean dataChanged) { return CompletableFuture.completedFuture(new DataFileUpdateResult(fullDataSource, dataChanged)); }
/** Can be used to update world gen queues or run any other data checking necessary when initially loading a file */
default void onRenderDataFileLoaded(DhSectionPos pos) { }
@Nullable
FullDataMetaFile getFileIfExist(DhSectionPos pos);
void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor chunkData);
FullDataRepo getRepo();
//================//
// helper classes //
//================//
/**
* After a {@link FullDataMetaFile} has been updated the {@link IFullDataSourceProvider} may also need to modify it. <br>
* This specifically happens during world generation.
*/
class DataFileUpdateResult
{
IFullDataSource fullDataSource;
boolean dataSourceChanged;
public DataFileUpdateResult(IFullDataSource fullDataSource, boolean dataSourceChanged)
{
this.fullDataSource = fullDataSource;
this.dataSourceChanged = dataSourceChanged;
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java
+1
View File
@@ -142,6 +142,7 @@ public abstract class AbstractMetaDataContainerFile
// helper classes //
//================//
/** TODO replace with a method that accepts a {@link DhDataOutputStream} and writes to that instead */
@FunctionalInterface
public interface IMetaDataWriterFunc<T> { void writeBinaryDataToStream(T t) throws IOException; }
core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java
+3 -2
View File
@@ -35,6 +35,8 @@ public class BaseMetaData
{
public DhSectionPos pos;
public int checksum;
/** @deprecated the database now has a last modified date time that should be used instead */
@Deprecated
public AtomicLong dataVersion = new AtomicLong(Long.MAX_VALUE);
public byte dataDetailLevel;
public EDhApiWorldGenerationStep worldGenStep;
@@ -46,11 +48,10 @@ public class BaseMetaData
public BaseMetaData(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, long dataVersion)
public BaseMetaData(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion)
{
this.pos = pos;
this.checksum = checksum;
this.dataVersion = new AtomicLong(dataVersion);
this.dataDetailLevel = dataDetailLevel;
this.worldGenStep = worldGenStep;
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java
+21 -28
View File
@@ -23,8 +23,6 @@ import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.file.DataSourceReferenceTracker;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
@@ -38,13 +36,13 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.MetaDataDto;
import com.seibel.distanthorizons.core.util.AtomicsUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.Reference;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.*;
import java.lang.ref.SoftReference;
import java.util.Random;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
@@ -64,7 +62,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
* When clearing, don't set to null, instead create a SoftReference containing null.
* This makes null checks simpler.
*/
private DataSourceReferenceTracker.RenderDataSourceSoftRef cachedRenderDataSourceRef = new DataSourceReferenceTracker.RenderDataSourceSoftRef(this, null);
private SoftReference<ColumnRenderSource> cachedRenderDataSourceRef = new SoftReference<>(null);
private final AtomicReference<CompletableFuture<ColumnRenderSource>> renderSourceLoadFutureRef = new AtomicReference<>(null);
private final IDhClientLevel clientLevel;
@@ -109,9 +107,6 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
LodUtil.assertTrue(this.baseMetaData != null);
this.doesDtoExist = true;
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderDataFileStatus);
// handles world gen queuing for missing columns
this.fullDataSourceProvider.onRenderDataFileLoaded(this.baseMetaData.pos);
}
@@ -196,11 +191,11 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
this.baseMetaData = new BaseMetaData(
newColumnRenderSource.getSectionPos(), -1, newColumnRenderSource.getDataDetailLevel(),
newColumnRenderSource.worldGenStep, RENDER_SOURCE_TYPE,
newColumnRenderSource.getRenderDataFormatVersion(), Long.MAX_VALUE);
newColumnRenderSource.getRenderDataFormatVersion());
this.updateRenderCacheAsync(newColumnRenderSource).whenComplete((voidObj, ex) ->
{
this.cachedRenderDataSourceRef = new DataSourceReferenceTracker.RenderDataSourceSoftRef(this, newColumnRenderSource);
this.cachedRenderDataSourceRef = new SoftReference<>(newColumnRenderSource);
this.renderSourceLoadFutureRef.set(null);
getSourceFuture.complete(newColumnRenderSource);
@@ -248,7 +243,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
this.renderSourceLoadFutureRef.set(null);
this.cachedRenderDataSourceRef = new DataSourceReferenceTracker.RenderDataSourceSoftRef(this, renderSource);
this.cachedRenderDataSourceRef = new SoftReference<>(renderSource);
getSourceFuture.complete(renderSource);
});
}
@@ -273,32 +268,30 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
DebugRenderer.BoxWithLife debugBox = new DebugRenderer.BoxWithLife(new DebugRenderer.Box(renderSource.sectionPos, 74f, 86f, 0.1f, Color.red), 1.0, 32f, Color.green.darker());
// Skip updating the cache if the data file is already up-to-date
FullDataMetaFile dataFile = this.fullDataSourceProvider.getFileIfExist(this.pos);
if (!ALWAYS_INVALIDATE_CACHE && dataFile != null && dataFile.baseMetaData != null && dataFile.baseMetaData.checksum == this.baseMetaData.dataVersion.get()) // TODO can we make it so the version comparisons either both use the checksum or the dataVersion? Comparing checksum and dataVersion is kinda confusing
{
LOGGER.debug("Skipping render cache update for " + this.pos);
renderSource.localVersion.incrementAndGet();
return CompletableFuture.completedFuture(renderSource);
}
//// Skip updating the cache if the data file is already up-to-date
//FullDataMetaFile dataFile = this.fullDataSourceProvider.getDtoIfExist(this.pos);
//if (!ALWAYS_INVALIDATE_CACHE && dataFile != null && dataFile.baseMetaData != null && dataFile.baseMetaData.checksum == this.baseMetaData.dataVersion.get()) // TODO can we make it so the version comparisons either both use the checksum or the dataVersion? Comparing checksum and dataVersion is kinda confusing
//{
// LOGGER.debug("Skipping render cache update for " + this.pos);
// renderSource.localVersion.incrementAndGet();
// return CompletableFuture.completedFuture(renderSource);
//}
final Reference<Integer> renderDataVersionRef = new Reference<>(Integer.MAX_VALUE);
// get the full data source
CompletableFuture<IFullDataSource> fullDataSourceFuture =
this.fullDataSourceProvider.readAsync(renderSource.getSectionPos())
this.fullDataSourceProvider.getAsync(renderSource.getSectionPos())
.thenApply((fullDataSource) ->
{
debugBox.box.color = Color.yellow.darker();
// get the metaFile's version
FullDataMetaFile renderSourceMetaFile = this.fullDataSourceProvider.getFileIfExist(this.pos);
if (renderSourceMetaFile != null && renderSourceMetaFile.baseMetaData != null)
{
renderDataVersionRef.value = renderSourceMetaFile.baseMetaData.checksum;
}
//// get the metaFile's version
//FullDataMetaFile renderSourceMetaFile = this.fullDataSourceProvider.getDtoIfExist(this.pos);
//if (renderSourceMetaFile != null && renderSourceMetaFile.baseMetaData != null)
//{
// renderDataVersionRef.value = renderSourceMetaFile.baseMetaData.checksum;
//}
return fullDataSource;
}).exceptionally((ex) ->
@@ -332,7 +325,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
renderSource.updateFromRenderSource(newRenderSource);
// update the meta data
this.baseMetaData.dataVersion.set(renderDataVersionRef.value);
this.baseMetaData.dataVersion.set(Integer.MAX_VALUE);
this.baseMetaData.dataDetailLevel = renderSource.getDataDetailLevel();
this.baseMetaData.dataType = RENDER_SOURCE_TYPE;
this.baseMetaData.binaryDataFormatVersion = renderSource.getRenderDataFormatVersion();
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java
+1 -1
View File
@@ -213,7 +213,7 @@ public class RenderSourceFileHandler implements IRenderSourceProvider
{
// convert to the lowest detail level so all detail levels are updated
this.writeChunkDataToFileRecursively(chunkDataView, DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
this.fullDataSourceProvider.writeChunkDataToFile(sectionPos, chunkDataView);
this.fullDataSourceProvider.updateDataSourcesWithChunkData(chunkDataView);
}
private void writeChunkDataToFileRecursively(ChunkSizedFullDataAccessor chunk, byte sectionDetailLevel)
{
core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java
+1 -1
View File
@@ -219,7 +219,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable
{
// get the data source to compare against
IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), this.currentClientLevel, testLevelFolder, false);
IFullDataSource testFullDataSource = tempLevel.getFileHandler().readAsync(new DhSectionPos(this.playerData.playerBlockPos)).join();
IFullDataSource testFullDataSource = tempLevel.getFileHandler().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join();
if (testFullDataSource == null)
{
continue;
core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java
-1
View File
@@ -35,7 +35,6 @@ public interface IWorldGenerationQueue extends Closeable
byte highestDataDetail();
CompletableFuture<WorldGenResult> submitGenTask(DhSectionPos pos, byte requiredDataDetail, IWorldGenTaskTracker tracker);
void cancelGenTasks(Iterable<DhSectionPos> positions);
/** @param targetPos the position that world generation should be centered around, generally this will be the player's position. */
void startGenerationQueueAndSetTargetPos(DhBlockPos2D targetPos);
core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java
-6
View File
@@ -159,12 +159,6 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender
return future;
}
@Override
public void cancelGenTasks(Iterable<DhSectionPos> positions)
{
// TODO Should we cancel generation of chunks that were loaded by the player?
}
//===============//
core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java
+1 -14
View File
@@ -189,20 +189,7 @@ public class ClientLevelModule implements Closeable
}
else
{
this.parentClientLevel.getFileHandler().writeChunkDataToFile(pos, data);
}
}
public CompletableFuture<Void> saveAsync()
{
ClientRenderState ClientRenderState = this.ClientRenderStateRef.get();
if (ClientRenderState != null)
{
return ClientRenderState.renderSourceFileHandler.flushAndSaveAsync();
}
else
{
return CompletableFuture.completedFuture(null);
this.parentClientLevel.getFileHandler().updateDataSourcesWithChunkData(data);
}
}
core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java
-6
View File
@@ -115,12 +115,6 @@ public class DhClientLevel extends DhLevel implements IDhClientLevel
@Override
public ILevelWrapper getLevelWrapper() { return levelWrapper; }
@Override
public CompletableFuture<Void> saveAsync()
{
return CompletableFuture.allOf(clientside.saveAsync(), dataFileHandler.flushAndSaveAsync());
}
@Override
public void saveWrites(ChunkSizedFullDataAccessor data) { this.clientside.writeChunkDataToFile(data); }
core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java
+19 -9
View File
@@ -22,12 +22,14 @@ package com.seibel.distanthorizons.core.level;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
import com.seibel.distanthorizons.core.render.LodRenderSection;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhBlockPos;
import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
@@ -39,7 +41,7 @@ import com.seibel.distanthorizons.coreapi.util.math.Mat4f;
import org.apache.logging.log4j.Logger;
import java.awt.*;
import java.util.concurrent.CompletableFuture;
import java.util.Iterator;
/** The level used on a singleplayer world */
public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhServerLevel
@@ -90,25 +92,37 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS
@Override
public void doWorldGen()
{
this.serverside.worldGeneratorEnabledConfig.pollNewValue();
this.serverside.worldGeneratorEnabledConfig.pollNewValue(); // if not called the get() line below may not
boolean shouldDoWorldGen = this.serverside.worldGeneratorEnabledConfig.get() && this.clientside.isRendering();
boolean isWorldGenRunning = this.serverside.worldGenModule.isWorldGenRunning();
if (shouldDoWorldGen && !isWorldGenRunning)
{
// start world gen
// create a new queue
this.serverside.worldGenModule.startWorldGen(this.serverside.dataFileHandler, new ServerLevelModule.WorldGenState(this));
// populate the queue based on the current rendering tree
ClientLevelModule.ClientRenderState renderState = this.clientside.ClientRenderStateRef.get();
Iterator<QuadNode<LodRenderSection>> iterator = renderState.quadtree.leafNodeIterator();
while (iterator.hasNext())
{
QuadNode<LodRenderSection> node = iterator.next();
this.serverside.dataFileHandler.getAsync(node.sectionPos);
}
}
else if (!shouldDoWorldGen && isWorldGenRunning)
{
// stop world gen
this.serverside.worldGenModule.stopWorldGen(this.serverside.dataFileHandler);
}
if (this.serverside.worldGenModule.isWorldGenRunning())
if (isWorldGenRunning)
{
ClientLevelModule.ClientRenderState renderState = this.clientside.ClientRenderStateRef.get();
if (renderState != null && renderState.quadtree != null)
{
// remove any generator sections that are out of bounds
this.serverside.dataFileHandler.removeGenRequestIf(pos -> !renderState.quadtree.isSectionPosInBounds(pos));
}
@@ -177,11 +191,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS
@Override
public int getMinY() { return getLevelWrapper().getMinHeight(); }
@Override
public CompletableFuture<Void> saveAsync()
{
return CompletableFuture.allOf(clientside.saveAsync(), getFileHandler().flushAndSaveAsync());
}
//===============//
// data handling //
core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java
+1 -6
View File
@@ -30,8 +30,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper;
import org.apache.logging.log4j.Logger;
import java.util.concurrent.CompletableFuture;
public class DhServerLevel extends DhLevel implements IDhServerLevel
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@@ -56,7 +54,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
{
DhSectionPos pos = data.getSectionPos();
pos = pos.convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET);
this.getFileHandler().writeChunkDataToFile(pos, data);
this.getFileHandler().updateDataSourcesWithChunkData(data);
}
@Override
@@ -70,9 +68,6 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
LOGGER.info("Closed DHLevel for {}", getLevelWrapper());
}
@Override
public CompletableFuture<Void> saveAsync() { return getFileHandler().flushAndSaveAsync(); }
@Override
public void doWorldGen()
{
core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java
-1
View File
@@ -29,7 +29,6 @@ import java.util.concurrent.CompletableFuture;
public interface IDhLevel extends AutoCloseable
{
int getMinY();
CompletableFuture<Void> saveAsync();
/**
* May return either a client or server level wrapper. <br>
core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java
+31 -1
View File
@@ -41,6 +41,10 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
public String getPrimaryKeyName() { return "DhSectionPos"; }
//=======================//
// repo required methods //
//=======================//
@Override
public MetaDataDto convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
{
@@ -59,7 +63,7 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
BaseMetaData baseMetaData = new BaseMetaData(pos,
checksum, dataDetailLevel, worldGenStep,
dataType, binaryDataFormatVersion, dataVersion);
dataType, binaryDataFormatVersion);
// binary data
byte[] dataByteArray = (byte[]) objectMap.get("Data");
@@ -137,4 +141,30 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
}
//=====================//
// data source methods //
//=====================//
/**
* Returns the highest numerical detail level in this table. <Br>
* Returns {@link DhSectionPos#SECTION_MINIMUM_DETAIL_LEVEL} if no data is present.
*/
public int getMaxSectionDetailLevel()
{
Map<String, Object> resultMap = this.queryDictionaryFirst("select MAX(DataDetailLevel) as maxDetailLevel from DhFullData;");
int maxDetailLevel;
if (resultMap == null || resultMap.get("maxDetailLevel") == null)
{
maxDetailLevel = 0;
}
else
{
maxDetailLevel = (int)resultMap.get("maxDetailLevel");
}
return maxDetailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java
+14
View File
@@ -21,7 +21,13 @@ package com.seibel.distanthorizons.core.sql;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */
public class MetaDataDto implements IBaseDTO
@@ -40,4 +46,12 @@ public class MetaDataDto implements IBaseDTO
@Override
public String getPrimaryKeyString() { return this.baseMetaData.pos.serialize(); }
/** @return a stream for the data contained in this DTO. */
public DhDataInputStream getInputStream() throws IOException
{
InputStream inputStream = new ByteArrayInputStream(this.dataArray);
DhDataInputStream compressedStream = new DhDataInputStream(inputStream);
return compressedStream;
}
}
core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java
-5
View File
@@ -151,15 +151,10 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor
public void doWorldGen() { this.dhLevels.forEach(DhClientServerLevel::doWorldGen); }
@Override
public CompletableFuture<Void> saveAndFlush() { return CompletableFuture.allOf(this.dhLevels.stream().map(DhClientServerLevel::saveAsync).toArray(CompletableFuture[]::new)); }
/** synchronized to prevent a rare issue where the server tries closing the same world multiple times in rapid succession. */
@Override
public synchronized void close()
{
// at this point the levels are probably unloaded, so this save call usually generally won't do anything
this.saveAndFlush();
this.f3Message.close();
core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java
-7
View File
@@ -173,12 +173,6 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld
// Not implemented
}
@Override
public CompletableFuture<Void> saveAndFlush()
{
return CompletableFuture.allOf(this.levels.values().stream().map(DhClientLevel::saveAsync).toArray(CompletableFuture[]::new));
}
@Override
public void close()
{
@@ -188,7 +182,6 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld
// }
this.saveAndFlush();
for (DhClientLevel dhClientLevel : this.levels.values())
{
LOGGER.info("Unloading level " + dhClientLevel.getLevelWrapper().getDimensionType().getDimensionName());
core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java
-6
View File
@@ -188,12 +188,6 @@ public class DhServerWorld extends AbstractDhWorld implements IDhServerWorld
public void doWorldGen() { this.levels.values().forEach(DhServerLevel::doWorldGen); }
@Override
public CompletableFuture<Void> saveAndFlush()
{
return CompletableFuture.allOf(this.levels.values().stream().map(DhServerLevel::saveAsync).toArray(CompletableFuture[]::new));
}
@Override
public void close()
{
core/src/main/java/com/seibel/distanthorizons/core/world/IDhWorld.java
-2
View File
@@ -34,6 +34,4 @@ public interface IDhWorld
void unloadLevel(@NotNull ILevelWrapper levelWrapper);
CompletableFuture<Void> saveAndFlush();
}