diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java index 2cae7d488..32eb41f19 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java @@ -32,7 +32,7 @@ public abstract class AbstractFullDataSourceLoader { public static final HashMultimap, AbstractFullDataSourceLoader> loaderRegistry = HashMultimap.create(); - public final Class clazz; + public final Class fullDataSourceClass; public static final HashMap> datatypeIdRegistry = new HashMap<>(); public final long datatypeId; @@ -40,18 +40,18 @@ public abstract class AbstractFullDataSourceLoader - public AbstractFullDataSourceLoader(Class clazz, long datatypeId, byte[] loaderSupportedVersions) + public AbstractFullDataSourceLoader(Class fullDataSourceClass, long datatypeId, byte[] loaderSupportedVersions) { this.datatypeId = datatypeId; this.loaderSupportedVersions = loaderSupportedVersions; Arrays.sort(loaderSupportedVersions); // sort to allow fast access - this.clazz = clazz; - if (datatypeIdRegistry.containsKey(datatypeId) && datatypeIdRegistry.get(datatypeId) != clazz) + this.fullDataSourceClass = fullDataSourceClass; + if (datatypeIdRegistry.containsKey(datatypeId) && datatypeIdRegistry.get(datatypeId) != fullDataSourceClass) { throw new IllegalArgumentException("Loader for datatypeId " + datatypeId + " already registered with different class: " - + datatypeIdRegistry.get(datatypeId) + " != " + clazz); + + datatypeIdRegistry.get(datatypeId) + " != " + fullDataSourceClass); } - Set loaders = loaderRegistry.get(clazz); + Set loaders = loaderRegistry.get(fullDataSourceClass); if (loaders.stream().anyMatch(other -> { // see if any loaderSupportsVersion conflicts with this one @@ -65,11 +65,11 @@ public abstract class AbstractFullDataSourceLoader return false; })) { - throw new IllegalArgumentException("Loader for class " + clazz + " that supports one of the version in " + throw new IllegalArgumentException("Loader for class " + fullDataSourceClass + " that supports one of the version in " + Arrays.toString(loaderSupportedVersions) + " already registered!"); } - datatypeIdRegistry.put(datatypeId, clazz); - loaderRegistry.put(clazz, this); + datatypeIdRegistry.put(datatypeId, fullDataSourceClass); + loaderRegistry.put(fullDataSourceClass, this); } /** diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 359543b40..232280b58 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -525,7 +525,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider //=========// @Override - public void close() { FullDataMetaFile.debugPhantomLifeCycleCheck(); } + public void close() { FullDataMetaFile.checkAndLogPhantomDataSourceLifeCycles(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java index 1246eb254..7a5cc1bae 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java @@ -45,82 +45,52 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import org.apache.logging.log4j.Logger; -/** - * Represents a File that contains a {@link IFullDataSource}. - */ +/** Represents a File that contains a {@link IFullDataSource}. */ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable { public static final String FILE_SUFFIX = ".lod"; private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName()); + // === Object lifetime tracking === + /** if true both data source creation and garbage collection will be logged */ + private static final boolean LOG_DATA_SOURCE_LIVES = false; + private static final ReferenceQueue LIFE_CYCLE_DEBUG_QUEUE = new ReferenceQueue<>(); + private static final ReferenceQueue SOFT_REF_DEBUG_QUEUE = new ReferenceQueue<>(); + private static final Set LIFE_CYCLE_DEBUG_SET = ConcurrentHashMap.newKeySet(); + private static final Set SOFT_REF_DEBUG_SET = ConcurrentHashMap.newKeySet(); + // =========================== + - private final IDhLevel level; - private final IFullDataSourceProvider fullDataSourceProvider; public boolean doesFileExist; - //TODO: Atm can't find a better way to store when genQueue is checked. public boolean genQueueChecked = false; + public AbstractFullDataSourceLoader fullDataSourceLoader; + public Class fullDataSourceClass; + + private volatile boolean markedNeedUpdate = false; - public AbstractFullDataSourceLoader fullDataSourceLoader; - public Class dataType; + private final IDhLevel level; + private final IFullDataSourceProvider fullDataSourceProvider; /** * Can be cleared if the garbage collector determines there isn't enough space.

* * When clearing, don't set to null, instead create a SoftReference containing null. - * This will make null checks simpler. + * This makes null checks simpler. */ private SoftReference cachedFullDataSourceRef = new SoftReference<>(null); private final AtomicReference> dataSourceLoadFutureRef = new AtomicReference<>(null); - - // ===Concurrent Write stuff=== + // === Concurrent Write tracking === private final AtomicReference writeQueueRef = new AtomicReference<>(new GuardedMultiAppendQueue()); private GuardedMultiAppendQueue backWriteQueue = new GuardedMultiAppendQueue(); // =========================== - // ===Object lifetime stuff=== - private static final ReferenceQueue lifeCycleDebugQueue = new ReferenceQueue<>(); - private static final ReferenceQueue softRefDebugQueue = new ReferenceQueue<>(); - private static final Set lifeCycleDebugSet = ConcurrentHashMap.newKeySet(); - private static final Set softRefDebugSet = ConcurrentHashMap.newKeySet(); - - private static class DataObjTracker extends PhantomReference implements Closeable - { - public final DhSectionPos pos; - DataObjTracker(IFullDataSource data) - { - super(data, lifeCycleDebugQueue); - //LOGGER.info("Phantom created on {}! count: {}", data.getSectionPos(), lifeCycleDebugSet.size()); - lifeCycleDebugSet.add(this); - this.pos = data.getSectionPos(); - } - @Override - public void close() { lifeCycleDebugSet.remove(this); } - - } - - private static class DataObjSoftTracker extends SoftReference implements Closeable - { - public final FullDataMetaFile file; - DataObjSoftTracker(FullDataMetaFile file, IFullDataSource data) - { - super(data, softRefDebugQueue); - softRefDebugSet.add(this); - this.file = file; - } - @Override - public void close() { softRefDebugSet.remove(this); } - - } - // =========================== - - //==============// // constructors // @@ -134,7 +104,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I private FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, DhSectionPos pos) throws IOException { super(fullDataSourceProvider.computeDataFilePath(pos), pos); - debugPhantomLifeCycleCheck(); + checkAndLogPhantomDataSourceLifeCycles(); this.fullDataSourceProvider = fullDataSourceProvider; this.level = level; @@ -153,7 +123,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I private FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, File file) throws IOException, FileNotFoundException { super(file); - debugPhantomLifeCycleCheck(); + checkAndLogPhantomDataSourceLifeCycles(); this.fullDataSourceProvider = fullDataSourceProvider; this.level = level; @@ -167,7 +137,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I throw new IOException("Invalid file: Data type loader not found: " + this.baseMetaData.dataTypeId + "(v" + this.baseMetaData.binaryDataFormatVersion + ")"); } - this.dataType = this.fullDataSourceLoader.clazz; + this.fullDataSourceClass = this.fullDataSourceLoader.fullDataSourceClass; DebugRenderer.register(this); } @@ -177,12 +147,14 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I // get data // //==========// - // Try get cached data source. Used for temp impl for re-queueing world gen tasks. - // (Read-only access! As writes should always be done async) - public IFullDataSource getCachedDataSourceNowOrNull() - { - debugPhantomLifeCycleCheck(); - return this.cachedFullDataSourceRef.get(); + /** + * Try get cached data source. Used for temp impl of re-queueing world gen tasks. + * (Read-only access! As writes should always be done async) + */ + public IFullDataSource getCachedDataSourceNowOrNull() + { + checkAndLogPhantomDataSourceLifeCycles(); + return this.cachedFullDataSourceRef.get(); } private void makeUpdateCompletionStage(CompletableFuture completer, CompletableFuture currentStage) @@ -204,6 +176,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I new DataObjTracker(fullDataSource); new DataObjSoftTracker(this, fullDataSource); } + //LOGGER.info("Updated file "+this.file); if (this.pos.sectionDetailLevel == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL) DebugRenderer.makeParticle( @@ -259,7 +232,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I public CompletableFuture getOrLoadCachedDataSourceAsync() { - debugPhantomLifeCycleCheck(); + checkAndLogPhantomDataSourceLifeCycles(); CompletableFuture dataSourceLoadFuture = this.getCachedDataSourceAsync(); if (dataSourceLoadFuture != null) @@ -397,7 +370,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I */ public void addToWriteQueue(ChunkSizedFullDataAccessor chunkAccessor) { - debugPhantomLifeCycleCheck(); + checkAndLogPhantomDataSourceLifeCycles(); DhLodPos chunkLodPos = new DhLodPos(LodUtil.CHUNK_DETAIL_LEVEL, chunkAccessor.pos.x, chunkAccessor.pos.z); @@ -427,7 +400,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I /** Applies any queued {@link ChunkSizedFullDataAccessor} to this metadata's {@link IFullDataSource} and writes the data to file. */ public CompletableFuture flushAndSaveAsync() { - debugPhantomLifeCycleCheck(); + checkAndLogPhantomDataSourceLifeCycles(); boolean isEmpty = this.writeQueueRef.get().queue.isEmpty() && !markedNeedUpdate; if (!isEmpty) { @@ -467,7 +440,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I this.fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getBinaryDataFormatVersion()); LodUtil.assertTrue(this.fullDataSourceLoader != null, "No loader for " + fullDataSource.getClass() + " (v" + fullDataSource.getBinaryDataFormatVersion() + ")"); - this.dataType = fullDataSource.getClass(); + this.fullDataSourceClass = fullDataSource.getClass(); this.baseMetaData.dataTypeId = (this.fullDataSourceLoader == null) ? 0 : this.fullDataSourceLoader.datatypeId; this.baseMetaData.binaryDataFormatVersion = fullDataSource.getBinaryDataFormatVersion(); @@ -528,24 +501,33 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I // debugging // //===========// - public static void debugPhantomLifeCycleCheck() + /** can be used to log when data sources have been garbage collected */ + public static void checkAndLogPhantomDataSourceLifeCycles() { - DataObjTracker phantom = (DataObjTracker) lifeCycleDebugQueue.poll(); - + DataObjTracker phantomRef = (DataObjTracker) LIFE_CYCLE_DEBUG_QUEUE.poll(); // wait for the tracker to be garbage collected(?) - while (phantom != null) + while (phantomRef != null) { - //LOGGER.info("Full Data at pos: "+phantom.pos+" has been freed. "+lifeCycleDebugSet.size()+" Full Data files remaining."); - phantom.close(); - phantom = (DataObjTracker) lifeCycleDebugQueue.poll(); + if (LOG_DATA_SOURCE_LIVES) + { + LOGGER.info("Full Data at pos: " + phantomRef.pos + " has been freed. [" + LIFE_CYCLE_DEBUG_SET.size() + "] Full Data sources remaining."); + } + + phantomRef.close(); + phantomRef = (DataObjTracker) LIFE_CYCLE_DEBUG_QUEUE.poll(); } - DataObjSoftTracker soft = (DataObjSoftTracker) softRefDebugQueue.poll(); - while (soft != null) + + DataObjSoftTracker softRef = (DataObjSoftTracker) SOFT_REF_DEBUG_QUEUE.poll(); + while (softRef != null) { - //LOGGER.info("Full Data at pos: "+soft.file.pos+" has been soft released."); - soft.close(); - soft = (DataObjSoftTracker) softRefDebugQueue.poll(); + if (LOG_DATA_SOURCE_LIVES) + { + LOGGER.info("Full Data at pos: " + softRef.file.pos + " has been soft released."); + } + + softRef.close(); + softRef = (DataObjSoftTracker) SOFT_REF_DEBUG_QUEUE.poll(); } } @@ -638,4 +620,46 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I } + /** used to debug data source soft reference garbage collection */ + private static class DataObjTracker extends PhantomReference implements Closeable + { + public final DhSectionPos pos; + + + DataObjTracker(IFullDataSource data) + { + super(data, LIFE_CYCLE_DEBUG_QUEUE); + + if (LOG_DATA_SOURCE_LIVES) + { + LOGGER.info("Phantom created on {}! count: {}", data.getSectionPos(), LIFE_CYCLE_DEBUG_SET.size()); + } + + LIFE_CYCLE_DEBUG_SET.add(this); + this.pos = data.getSectionPos(); + } + + @Override + public void close() { LIFE_CYCLE_DEBUG_SET.remove(this); } + + } + + /** used to debug data source soft reference garbage collection */ + private static class DataObjSoftTracker extends SoftReference implements Closeable + { + public final FullDataMetaFile file; + + + DataObjSoftTracker(FullDataMetaFile file, IFullDataSource data) + { + super(data, SOFT_REF_DEBUG_QUEUE); + SOFT_REF_DEBUG_SET.add(this); + this.file = file; + } + + @Override + public void close() { SOFT_REF_DEBUG_SET.remove(this); } + + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java index 0132d3836..d5e739570 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java @@ -52,6 +52,7 @@ import java.util.Random; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; +/** Represents a File that contains a {@link ColumnRenderSource}. */ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -65,7 +66,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements * Can be cleared if the garbage collector determines there isn't enough space.

* * When clearing, don't set to null, instead create a SoftReference containing null. - * This will make null checks simpler. + * This makes null checks simpler. */ private SoftReference cachedRenderDataSource = new SoftReference<>(null); private final AtomicReference> renderSourceLoadFutureRef = new AtomicReference<>(null);