From 2400d999a3cf12a952c564eac2766a446bc65b32 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 7 May 2023 17:54:54 -0500 Subject: [PATCH] Refactor FullDataMetaFile --- .../sources/CompleteFullDataSource.java | 4 +- .../HighDetailIncompleteFullDataSource.java | 4 +- .../LowDetailIncompleteFullDataSource.java | 4 +- .../render/ColumnRenderLoader.java | 2 +- .../fullDatafile/FullDataFileHandler.java | 6 +- .../file/fullDatafile/FullDataMetaFile.java | 540 ++++++++++-------- .../GeneratedFullDataFileHandler.java | 2 +- .../AbstractMetaDataContainerFile.java | 37 +- .../lod/core/file/metaData/BaseMetaData.java | 6 +- .../file/renderfile/RenderMetaDataFile.java | 8 +- .../renderfile/RenderSourceFileHandler.java | 10 +- 11 files changed, 345 insertions(+), 278 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java index be2e9b7ac..2130c9732 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -91,9 +91,9 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu int dataDetail = dataInputStream.readInt(); - if (dataDetail != dataFile.metaData.dataLevel) + if (dataDetail != dataFile.baseMetaData.dataLevel) { - throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.metaData.dataLevel)); + throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.baseMetaData.dataLevel)); } int width = dataInputStream.readInt(); diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java index 710a1a10a..4eb1b6c72 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java @@ -132,9 +132,9 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo LodUtil.assertTrue(dataFile.pos.sectionDetailLevel <= MAX_SECTION_DETAIL); int dataDetail = dataInputStream.readShort(); - if(dataDetail != dataFile.metaData.dataLevel) + if(dataDetail != dataFile.baseMetaData.dataLevel) { - throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); + throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.baseMetaData.dataLevel)); } // confirm that the detail level is correct diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java index 612f312c2..cca987676 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java @@ -106,9 +106,9 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp int dataDetail = dataInputStream.readInt(); - if(dataDetail != dataFile.metaData.dataLevel) + if(dataDetail != dataFile.baseMetaData.dataLevel) { - throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.metaData.dataLevel)); + throw new IOException(LodUtil.formatLog("Data level mismatch: "+dataDetail+" != "+dataFile.baseMetaData.dataLevel)); } int width = dataInputStream.readInt(); diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java index 50bbb3f64..9a1b11ed4 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/render/ColumnRenderLoader.java @@ -39,7 +39,7 @@ public class ColumnRenderLoader public ColumnRenderSource loadRenderSource(RenderMetaDataFile dataFile, BufferedInputStream bufferedInputStream, IDhLevel level) throws IOException { DataInputStream inputDataStream = new DataInputStream(bufferedInputStream); // DO NOT CLOSE - int dataFileVersion = dataFile.metaData.loaderVersion; + int dataFileVersion = dataFile.baseMetaData.binaryDataFormatVersion; switch (dataFileVersion) { diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java index 938494dd9..5bba31de9 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java @@ -258,7 +258,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider // future wrapper necessary in order to handle file read errors CompletableFuture futureWrapper = new CompletableFuture<>(); - metaFile.loadOrGetCachedAsync().exceptionally((e) -> + metaFile.loadOrGetCachedDataSourceAsync().exceptionally((e) -> { FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, metaFile, e); @@ -365,7 +365,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider for (FullDataMetaFile metaFile : existFiles) { - futures.add(metaFile.loadOrGetCachedAsync() + futures.add(metaFile.loadOrGetCachedDataSourceAsync() .thenAccept((data) -> { if (data != null) @@ -458,7 +458,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider @Override public void close() { - FullDataMetaFile.debugCheck(); + FullDataMetaFile.debugPhantomLifeCycleCheck(); // stop any existing file tasks fileReaderThread.shutdownNow(); diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java index b732eab38..e090b6ea1 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java @@ -3,16 +3,15 @@ package com.seibel.lod.core.file.fullDatafile; import java.io.*; import java.lang.ref.*; import java.nio.channels.ClosedByInterruptException; +import java.nio.file.FileAlreadyExistsException; import java.util.Set; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.lod.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.lod.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; -import com.seibel.lod.core.dependencyInjection.SingletonInjector; import com.seibel.lod.core.file.metaData.BaseMetaData; import com.seibel.lod.core.pos.DhLodPos; import com.seibel.lod.core.file.metaData.AbstractMetaDataContainerFile; @@ -21,43 +20,53 @@ import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.AtomicsUtil; import com.seibel.lod.core.util.LodUtil; -import com.seibel.lod.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.Logger; /** - * Related to the stored Blockstate/Biome ID data. + * Represents a File that contains a {@link IFullDataSource}. */ public class FullDataMetaFile extends AbstractMetaDataContainerFile { private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName()); - private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); + private final IDhLevel level; - private final IFullDataSourceProvider handler; + private final IFullDataSourceProvider fullDataSourceProvider; private boolean doesFileExist; - + + public AbstractFullDataSourceLoader fullDataSourceLoader; public Class dataType; - // The '?' type should either be: - // SoftReference, or - Non-dirty file that can be GCed - // CompletableFuture, or - File that is being loaded. No guarantee that the type is promotable or not - // null - Nothing is loaded or being loaded + + /** + * Deprecated: this should be split up into multiple variables to prevent datatype confusion + * + * The '?' type should either be: + * SoftReference, or - Non-dirty file that can be GCed + * CompletableFuture, or - File that is being loaded. No guarantee that the type is promotable or not + * null - Nothing is loaded or being loaded + */ + @Deprecated AtomicReference data = new AtomicReference<>(null); - + + + //TODO: use ConcurrentAppendSingleSwapContainer instead of below: - private static class GuardedMultiAppendQueue { + private static class GuardedMultiAppendQueue + { ReentrantReadWriteLock appendLock = new ReentrantReadWriteLock(); ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); } - + + // ===Concurrent Write stuff=== - AtomicReference writeQueue = - new AtomicReference<>(new GuardedMultiAppendQueue()); - GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue(); + private final AtomicReference writeQueueRef = new AtomicReference<>(new GuardedMultiAppendQueue()); + private GuardedMultiAppendQueue backWriteQueue = new GuardedMultiAppendQueue(); // =========================== - - private AtomicReference> inCacheWriteAccessFuture = new AtomicReference<>(null); - + + + private final AtomicReference> inCacheWriteAccessFuture = new AtomicReference<>(null); + // ===Object lifetime stuff=== private static final ReferenceQueue lifeCycleDebugQueue = new ReferenceQueue<>(); private static final Set lifeCycleDebugSet = ConcurrentHashMap.newKeySet(); @@ -81,128 +90,94 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile - // Create a new metaFile - public FullDataMetaFile(IFullDataSourceProvider handler, IDhLevel level, DhSectionPos pos) throws IOException + //==============// + // constructors // + //==============// + + /** + * Creates a new file. + * @throws FileAlreadyExistsException if a file already exists. + */ + public FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, DhSectionPos pos) throws FileAlreadyExistsException { - super(handler.computeDataFilePath(pos), pos); - debugCheck(); - this.handler = handler; - this.level = level; - LodUtil.assertTrue(metaData == null); - doesFileExist = false; - } - - public FullDataMetaFile(IFullDataSourceProvider handler, IDhLevel level, File path) throws IOException - { - super(path); - debugCheck(); - this.handler = handler; - this.level = level; - LodUtil.assertTrue(metaData != null); - fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(metaData.dataTypeId, metaData.loaderVersion); - if (fullDataSourceLoader == null) { - throw new IOException("Invalid file: Data type loader not found: " - + metaData.dataTypeId + "(v" + metaData.loaderVersion + ")"); - } - dataType = fullDataSourceLoader.clazz; - doesFileExist = true; - } - - - - - public CompletableFuture flushAndSaveAsync() - { - debugCheck(); - boolean isEmpty = this.writeQueue.get().queue.isEmpty(); - if (!isEmpty) - { - return this.loadOrGetCachedAsync().thenApply((unused) -> null); // This will flush the data to disk. - } - else - { - return CompletableFuture.completedFuture(null); - } - } - -// public long getCacheVersion() { -// debugCheck(); -// return (this.metaData == null) ? 0 : this.metaData.dataVersion.get(); -// } - -// public boolean isCacheVersionValid(long cacheVersion) -// { -// debugCheck(); -// boolean noWrite = this.writeQueue.get().queue.isEmpty(); -// if (!noWrite) -// { -// return false; -// } -// else -// { -// BaseMetaData getData = this.metaData; -// //NOTE: Do this instead of direct compare so values that wrapped around still work correctly. -// return (getData == null ? 0 : this.metaData.dataVersion.get()) - cacheVersion <= 0; -// } -// } - - public void addToWriteQueue(ChunkSizedFullDataAccessor chunkDataSource) - { - debugCheck(); - DhLodPos chunkLodPos = new DhLodPos(LodUtil.CHUNK_DETAIL_LEVEL, chunkDataSource.pos.x, chunkDataSource.pos.z); - LodUtil.assertTrue(pos.getSectionBBoxPos().overlapsExactly(chunkLodPos), "Chunk pos "+chunkLodPos+" doesn't overlap with section "+pos); - //LOGGER.info("Write Chunk {} to file {}", chunkPos, pos); + super(fullDataSourceProvider.computeDataFilePath(pos), pos); + debugPhantomLifeCycleCheck(); - GuardedMultiAppendQueue writeQueue = this.writeQueue.get(); - // Using read lock is OK, because the queue's underlying data structure is thread-safe. - // This lock is only used to insure on polling the queue, that the queue is not being - // modified by another thread. - Lock appendLock = writeQueue.appendLock.readLock(); - appendLock.lock(); - try + this.fullDataSourceProvider = fullDataSourceProvider; + this.level = level; + LodUtil.assertTrue(this.baseMetaData == null); + this.doesFileExist = false; + } + + /** + * Uses an existing file. + * @throws IOException if the file was formatted incorrectly + * @throws FileNotFoundException if no file exists for the given path + */ + public FullDataMetaFile(IFullDataSourceProvider fullDataSourceProvider, IDhLevel level, File file) throws IOException, FileNotFoundException + { + super(file); + debugPhantomLifeCycleCheck(); + + this.fullDataSourceProvider = fullDataSourceProvider; + this.level = level; + LodUtil.assertTrue(this.baseMetaData != null); + this.doesFileExist = true; + + this.fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(this.baseMetaData.dataTypeId, this.baseMetaData.binaryDataFormatVersion); + if (this.fullDataSourceLoader == null) { - writeQueue.queue.add(chunkDataSource); - } - finally - { - appendLock.unlock(); + throw new IOException("Invalid file: Data type loader not found: "+this.baseMetaData.dataTypeId+"(v"+this.baseMetaData.binaryDataFormatVersion +")"); } - //LOGGER.info("write queue length for pos "+this.pos+": " + writeQueue.queue.size()); + this.dataType = this.fullDataSourceLoader.clazz; } + + + //==========// + // get data // + //==========// + // Cause: Generic Type runtime casting cannot safety check it. // However, the Union type ensures the 'data' should only contain the listed type. - public CompletableFuture loadOrGetCachedAsync() + public CompletableFuture loadOrGetCachedDataSourceAsync() { - debugCheck(); + debugPhantomLifeCycleCheck(); Object obj = this.data.get(); - CompletableFuture cached = this._readCachedAsync(obj); + CompletableFuture cached = this.getCachedDataSourceAsync(obj); if (cached != null) { return cached; } + + CompletableFuture future = new CompletableFuture<>(); // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( - boolean worked = this.data.compareAndSet(obj, future); // TODO obj and future are different object types, would this ever return true? + boolean worked = this.data.compareAndSet(obj, future); // TODO if data was a future it would have a different memory address, would this ever return true? if (!worked) { - return this.loadOrGetCachedAsync(); + // TODO wouldn't this cause an infinite loop? + return this.loadOrGetCachedDataSourceAsync(); } + + // After cas. We are in exclusive control. if (!this.doesFileExist) { - this.handler.onCreateDataFile(this) + // create a new Meta file + + this.fullDataSourceProvider.onCreateDataFile(this) .thenApply((data) -> { - this.metaData = makeMetaData(data); + this.baseMetaData = this._makeBaseMetaData(data); return data; }) - .thenApply((data) -> this.handler.onDataFileLoaded(data, this.metaData, this::saveChanges, this::applyWriteQueue)) + .thenApply((data) -> this.fullDataSourceProvider.onDataFileLoaded(data, this.baseMetaData, this::_updateAndWriteDataSource, this::_applyWriteQueueToFullDataSource)) .whenComplete((fullDataSource, exception) -> { if (exception != null) @@ -221,16 +196,19 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile } else { + // read in the existing meta file's data + + if (this.baseMetaData == null) + { + throw new IllegalStateException("Meta data not loaded!"); + } + + CompletableFuture.supplyAsync(() -> { - if (this.metaData == null) - { - throw new IllegalStateException("Meta data not loaded!"); // TODO should this be a CompletionException? - } - // Load the file. IFullDataSource fullDataSource; - try (FileInputStream fileInputStream = this.getFileInputStream(); + try (FileInputStream fileInputStream = this._getFileInputStream(); BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { fullDataSource = this.fullDataSourceLoader.loadData(this, bufferedInputStream, this.level); @@ -247,31 +225,34 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile throw new CompletionException(ex); } - // Apply the write queue + + // confirm that this thread is in control LodUtil.assertTrue(this.inCacheWriteAccessFuture.get() == null, "No one should be writing to the cache while we are in the process of " + "loading one into the cache! Is this a deadlock?"); - fullDataSource = this.handler.onDataFileLoaded(fullDataSource, this.metaData, this::saveChanges, this::applyWriteQueue); + // fire the onDataLoaded method + fullDataSource = this.fullDataSourceProvider.onDataFileLoaded(fullDataSource, this.baseMetaData, this::_updateAndWriteDataSource, this::_applyWriteQueueToFullDataSource); return fullDataSource; - }, this.handler.getIOExecutor()) + + }, this.fullDataSourceProvider.getIOExecutor()) .exceptionally((ex) -> { if (ex instanceof InterruptedException) { + // this exception can be ignored //LOGGER.warn(FullDataMetaFile.class.getSimpleName()+" loadOrGetCachedAsync interrupted."); - //future.completeExceptionally(ex); // this exception can be ignored return null; } else if (ex instanceof RejectedExecutionException) { + // this exception can be ignored //LOGGER.warn(FullDataMetaFile.class.getSimpleName()+" loadOrGetCachedAsync attempted to use a closed thread pool."); - //future.completeExceptionally(ex); // this exception can be ignored return null; } - LOGGER.error("Error loading file {}: ", this.file, ex); + LOGGER.error("Error loading file "+this.file+": ", ex); this.data.set(null); future.completeExceptionally(ex); @@ -285,127 +266,212 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile }); } - // Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :( - //return future.completeAsync(this::loadAndUpdateDataSource, fileReaderThreads); + return future; } - - private static BaseMetaData makeMetaData(IFullDataSource data) { + /** @return a stream for the data contained in this file, skips the metadata from {@link AbstractMetaDataContainerFile}. */ + private FileInputStream _getFileInputStream() throws IOException + { + FileInputStream fileInputStream = new FileInputStream(this.file); + + // skip the meta-data bytes + int bytesToSkip = AbstractMetaDataContainerFile.METADATA_SIZE_IN_BYTES; + while (bytesToSkip > 0) + { + long skippedByteCount = fileInputStream.skip(bytesToSkip); + if (skippedByteCount == 0) + { + throw new IOException("Invalid file: Failed to skip metadata."); + } + bytesToSkip -= skippedByteCount; + } + + if (bytesToSkip != 0) + { + throw new IOException("File IO Error: Failed to skip metadata."); + } + return fileInputStream; + } + private BaseMetaData _makeBaseMetaData(IFullDataSource data) + { AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(data.getClass(), data.getBinaryDataFormatVersion()); return new BaseMetaData(data.getSectionPos(), -1, data.getDataDetailLevel(), data.getWorldGenStep(), (loader == null ? 0 : loader.datatypeId), data.getBinaryDataFormatVersion()); } - - // "unchecked": Suppress casting of CompletableFuture to CompletableFuture - // "PointlessBooleanExpression": Suppress explicit (boolean == false) check for more understandable CAS operation code. - private CompletableFuture _readCachedAsync(Object obj) + /** + * @return either the cached {@link IFullDataSource}, + * a future that will complete once the {@link FullDataMetaFile#writeQueueRef} has been written, + * or null if something went wrong + */ + private CompletableFuture getCachedDataSourceAsync(Object obj) { - // Has file cached in RAM and not freed yet. if ((obj instanceof SoftReference)) { - Object inner = ((SoftReference)obj).get(); - if (inner != null) + // The file is cached in RAM + + IFullDataSource innerFullDataSource = (IFullDataSource) ((SoftReference)obj).get(); + if (innerFullDataSource != null) { - LodUtil.assertTrue(inner instanceof IFullDataSource); - boolean isEmpty = writeQueue.get().queue.isEmpty(); + boolean writeQueueEmpty = this.writeQueueRef.get().queue.isEmpty(); + // If the queue is empty, and the CAS on inCacheWriteLock succeeds, then we are the thread // that will be applying the changes to the cache. - if (!isEmpty) + if (writeQueueEmpty) { - // Do a CAS on inCacheWriteLock to ensure that we are the only thread that is writing to the cache, - // or if we fail, then that means someone else is already doing it, and we can just return the future - CompletableFuture future = new CompletableFuture<>(); - CompletableFuture compareAndSwapFuture = AtomicsUtil.compareAndExchange(inCacheWriteAccessFuture, null, future); - if (compareAndSwapFuture == null) - { - try - { - data.set(future); - handler.onDataFileRefresh((IFullDataSource) inner, metaData, this::applyWriteQueue, this::saveChanges).handle((fullDataSource, exception) -> - { - if (exception != null) - { - LOGGER.error("Error refreshing data "+pos+": "+exception); - future.complete(null); - data.set(null); - } - else - { - future.complete(fullDataSource); - new DataObjTracker(fullDataSource); - data.set(new SoftReference<>(fullDataSource)); - } - inCacheWriteAccessFuture.set(null); - return fullDataSource; - }); - return future; - } - catch (Exception e) - { - LOGGER.error("Error while doing refreshes to LodDataSource at " + pos + ": " + e); - return CompletableFuture.completedFuture((IFullDataSource) inner); - } - } - else - { - // or, return the future that will be completed when the write is done. - return compareAndSwapFuture; - } + // return the cached data + return CompletableFuture.completedFuture(innerFullDataSource); } else { - // or, return the cached data. - return CompletableFuture.completedFuture((IFullDataSource) inner); + // either write the queue or return the future that is waiting for the queue write + + // Do a CAS on inCacheWriteLock to ensure that we are the only thread that is writing to the cache, + // or if we fail, then that means someone else is already doing it, and we can just return the future + CompletableFuture future = new CompletableFuture<>(); + CompletableFuture compareAndSwapFuture = AtomicsUtil.compareAndExchange(this.inCacheWriteAccessFuture, null, future); + if (compareAndSwapFuture != null) + { + // a write is already in progress, return its future. + return compareAndSwapFuture; + } + else + { + // write the queue to the data source + +// try // TODO is this try necessary? +// { + this.data.set(future); + + this.fullDataSourceProvider.onDataFileRefresh(innerFullDataSource, this.baseMetaData, this::_applyWriteQueueToFullDataSource, this::_updateAndWriteDataSource) + .handle((fullDataSource, exception) -> + { + if (exception != null) + { + LOGGER.error("Error refreshing data "+this.pos+": "+exception+" "+exception.getMessage()); + future.complete(null); + this.data.set(null); + } + else + { + future.complete(fullDataSource); + new DataObjTracker(fullDataSource); + this.data.set(new SoftReference<>(fullDataSource)); + } + + this.inCacheWriteAccessFuture.set(null); + return fullDataSource; + }); + return future; +// } +// catch (Exception e) +// { +// LOGGER.error("Error while doing refreshes to LodDataSource at "+this.pos+": "+e); +// return CompletableFuture.completedFuture(innerFullDataSource); +// } + } } } } - //==== Cached file out of scrope. ==== - // Someone is already trying to complete it. so just return the obj. - if ((obj instanceof CompletableFuture)) { - return (CompletableFuture)obj; + + //==== Cached file out of scope ==== + // Someone is already trying to complete it. so return the in-progress future. + if ((obj instanceof CompletableFuture)) + { + return (CompletableFuture) obj; } + + return null; } - - private void swapWriteQueue() { - GuardedMultiAppendQueue queue = writeQueue.getAndSet(_backQueue); - // Acquire write lock and then release it again as we only need to ensure that the queue - // is not being appended to by another thread. Note that the above atomic swap & - // the guarantee that all append first acquire the appendLock means after the locK() call, - // there will be no other threads able to or is currently appending to the queue. - // Note: The above needs the getAndSet() to have at least Release Memory order. - // (not that java supports anything non volatile for getAndSet()...) - queue.appendLock.writeLock().lock(); - queue.appendLock.writeLock().unlock(); - _backQueue = queue; + + + + //===============// + // data updating // + //===============// + + /** + * Adds the given {@link ChunkSizedFullDataAccessor} to the write queue, + * which will be applied to the object at some undefined time in the future. + */ + public void addToWriteQueue(ChunkSizedFullDataAccessor chunkAccessor) + { + debugPhantomLifeCycleCheck(); + + DhLodPos chunkLodPos = new DhLodPos(LodUtil.CHUNK_DETAIL_LEVEL, chunkAccessor.pos.x, chunkAccessor.pos.z); + + LodUtil.assertTrue(this.pos.getSectionBBoxPos().overlapsExactly(chunkLodPos), "Chunk pos "+chunkLodPos+" doesn't exactly overlap with section "+this.pos); + //LOGGER.info("Write Chunk {} to file {}", chunkPos, pos); + + GuardedMultiAppendQueue writeQueue = this.writeQueueRef.get(); + // Using read lock is OK, because the queue's underlying data structure is thread-safe. + // This lock is only used to insure on polling the queue, that the queue is not being + // modified by another thread. + ReentrantReadWriteLock.ReadLock appendLock = writeQueue.appendLock.readLock(); + appendLock.lock(); + try + { + writeQueue.queue.add(chunkAccessor); + } + finally + { + appendLock.unlock(); + } + + //LOGGER.info("write queue length for pos "+this.pos+": " + writeQueue.queue.size()); } - private void saveChanges(IFullDataSource fullDataSource) + + /** Applies any queued {@link ChunkSizedFullDataAccessor} to this metadata's {@link IFullDataSource} and writes the data to file. */ + public CompletableFuture flushAndSaveAsync() { - if (fullDataSource.isEmpty()) + debugPhantomLifeCycleCheck(); + boolean isEmpty = this.writeQueueRef.get().queue.isEmpty(); + if (!isEmpty) { - if (file.exists() && !file.delete()) - { - LOGGER.warn("Failed to delete data file at {}", file); - } - doesFileExist = false; + // This will flush the data to disk. + return this.loadOrGetCachedDataSourceAsync().thenApply((fullDataSource) -> null /* ignore the result, just wait for the load to finish*/); } else { + return CompletableFuture.completedFuture(null); + } + } + + + /** updates this object to match the given {@link IFullDataSource} and then writes the new data to file. */ + private void _updateAndWriteDataSource(IFullDataSource fullDataSource) + { + if (fullDataSource.isEmpty()) + { + // delete the empty data source + if (this.file.exists() && !this.file.delete()) + { + LOGGER.warn("Failed to delete data file at "+this.file); + } + this.doesFileExist = false; + } + else + { + // update the data source and write the new data to file + //LOGGER.info("Saving data file of {}", data.getSectionPos()); try { // Write/Update data - LodUtil.assertTrue(metaData != null); - metaData.dataLevel = fullDataSource.getDataDetailLevel(); - fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getBinaryDataFormatVersion()); - LodUtil.assertTrue(fullDataSourceLoader != null, "No loader for "+fullDataSource.getClass()+" (v"+fullDataSource.getBinaryDataFormatVersion()+")"); - dataType = fullDataSource.getClass(); - metaData.dataTypeId = (fullDataSourceLoader == null) ? 0 : fullDataSourceLoader.datatypeId; - metaData.loaderVersion = fullDataSource.getBinaryDataFormatVersion(); - super.writeData((outputStream) -> fullDataSource.writeToStream(outputStream, level)); - doesFileExist = true; + LodUtil.assertTrue(this.baseMetaData != null); + + this.baseMetaData.dataLevel = fullDataSource.getDataDetailLevel(); + this.fullDataSourceLoader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getBinaryDataFormatVersion()); + LodUtil.assertTrue(this.fullDataSourceLoader != null, "No loader for "+fullDataSource.getClass()+" (v"+fullDataSource.getBinaryDataFormatVersion()+")"); + + this.dataType = fullDataSource.getClass(); + this.baseMetaData.dataTypeId = (this.fullDataSourceLoader == null) ? 0 : this.fullDataSourceLoader.datatypeId; + this.baseMetaData.binaryDataFormatVersion = fullDataSource.getBinaryDataFormatVersion(); + + super.writeData((outputStream) -> fullDataSource.writeToStream(outputStream, this.level)); + this.doesFileExist = true; } catch (ClosedByInterruptException e) // thrown by buffers that are interrupted { @@ -414,55 +480,53 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile } catch (IOException e) { - LOGGER.error("Failed to save updated data file at "+file+" for sect "+pos, e); + LOGGER.error("Failed to save updated data file at "+this.file+" for section "+this.pos, e); } } } - /** @return whether any writing has happened to the data */ - private boolean applyWriteQueue(IFullDataSource fullDataSource) + /** @return true if the queue was not empty and data was applied to the {@link IFullDataSource}. */ + private boolean _applyWriteQueueToFullDataSource(IFullDataSource fullDataSource) { + // TODO this isn't being called enough + // Poll the write queue // First check if write queue is empty, then swap the write queue. // Must be done in this order to ensure isMemoryAddressValid work properly. See isMemoryAddressValid() for details. - boolean isEmpty = this.writeQueue.get().queue.isEmpty(); + boolean isEmpty = this.writeQueueRef.get().queue.isEmpty(); if (!isEmpty) { - this.swapWriteQueue(); - int count = this._backQueue.queue.size(); - for (ChunkSizedFullDataAccessor chunk : this._backQueue.queue) + this._swapWriteQueue(); + for (ChunkSizedFullDataAccessor chunk : this.backWriteQueue.queue) { fullDataSource.update(chunk); } - this._backQueue.queue.clear(); + this.backWriteQueue.queue.clear(); //LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count); } return !isEmpty; } - - private FileInputStream getFileInputStream() throws IOException + private void _swapWriteQueue() { - FileInputStream fileInputStream = new FileInputStream(this.file); - int toSkip = METADATA_SIZE_IN_BYTES; - while (toSkip > 0) - { - long skipped = fileInputStream.skip(toSkip); - if (skipped == 0) - { - throw new IOException("Invalid file: Failed to skip metadata."); - } - toSkip -= skipped; - } - - if (toSkip != 0) - { - throw new IOException("File IO Error: Failed to skip metadata."); - } - return fileInputStream; + GuardedMultiAppendQueue writeQueue = this.writeQueueRef.getAndSet(this.backWriteQueue); + // Acquire write lock and then release it again as we only need to ensure that the queue + // is not being appended to by another thread. Note that the above atomic swap & + // the guarantee that all append first acquire the appendLock means after the locK() call, + // there will be no other threads able to or is currently appending to the queue. + // Note: The above needs the getAndSet() to have at least Release Memory order. + // (not that java supports anything non volatile for getAndSet()...) + writeQueue.appendLock.writeLock().lock(); + writeQueue.appendLock.writeLock().unlock(); + this.backWriteQueue = writeQueue; } - public static void debugCheck() + + //===========// + // debugging // + //===========// + + public static void debugPhantomLifeCycleCheck() { DataObjTracker phantom = (DataObjTracker) lifeCycleDebugQueue.poll(); diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java index f1768ce60..7491f2034 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -145,7 +145,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler final ArrayList> loadDataFutures = new ArrayList<>(existingFiles.size()); for (FullDataMetaFile existingFile : existingFiles) { - loadDataFutures.add(existingFile.loadOrGetCachedAsync() + loadDataFutures.add(existingFile.loadOrGetCachedDataSourceAsync() .exceptionally((ex) -> /*Ignore file read errors*/null) .thenAccept((fullDataSource) -> { diff --git a/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java b/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java index ea5dbcde9..20ed50bdd 100644 --- a/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java @@ -73,9 +73,9 @@ public abstract class AbstractMetaDataContainerFile * Will be null if no file exists for this object.
* NOTE: Only use {@link BaseMetaData#pos} when initially setting up this object, afterwards the standalone {@link AbstractMetaDataContainerFile#pos} should be used. */ - public volatile BaseMetaData metaData = null; + public volatile BaseMetaData baseMetaData = null; - /** Should be used instead of the position inside {@link AbstractMetaDataContainerFile#metaData} */ + /** Should be used instead of the position inside {@link AbstractMetaDataContainerFile#baseMetaData} */ public final DhSectionPos pos; public File file; @@ -86,8 +86,11 @@ public abstract class AbstractMetaDataContainerFile // constructors // //==============// - /** Create a metaFile in this path. If the path has a file, throws FileAlreadyExistsException */ - protected AbstractMetaDataContainerFile(File file, DhSectionPos pos) throws IOException + /** + * Create a metaFile in this path. + * @throws FileAlreadyExistsException If the path already has a file. + */ + protected AbstractMetaDataContainerFile(File file, DhSectionPos pos) throws FileAlreadyExistsException { this.file = file; this.pos = pos; @@ -98,7 +101,7 @@ public abstract class AbstractMetaDataContainerFile } /** - * Creates a {@link AbstractMetaDataContainerFile} with the file at the given path. + * Creates an {@link AbstractMetaDataContainerFile} with the file at the given path. * @throws IOException if the file was formatted incorrectly * @throws FileNotFoundException if no file exists for the given path */ @@ -111,8 +114,8 @@ public abstract class AbstractMetaDataContainerFile } validateMetaDataFile(this.file); - this.metaData = readMetaDataFromFile(file); - this.pos = this.metaData.pos; + this.baseMetaData = readMetaDataFromFile(file); + this.pos = this.baseMetaData.pos; } /** * Attempts to create a new {@link AbstractMetaDataContainerFile} from the given file. @@ -173,21 +176,21 @@ public abstract class AbstractMetaDataContainerFile if (!file.canWrite()) throw new IOException("File not writable"); } - /** Sets this object's {@link AbstractMetaDataContainerFile#metaData} using the set {@link AbstractMetaDataContainerFile#file} */ + /** Sets this object's {@link AbstractMetaDataContainerFile#baseMetaData} using the set {@link AbstractMetaDataContainerFile#file} */ protected void loadMetaData() throws IOException { validateMetaDataFile(this.file); - this.metaData = readMetaDataFromFile(this.file); - if (!this.metaData.pos.equals(this.pos)) + this.baseMetaData = readMetaDataFromFile(this.file); + if (!this.baseMetaData.pos.equals(this.pos)) { - LOGGER.warn("The file is from a different location than expected! Expected: ["+this.pos+"] but got ["+this.metaData.pos+"]. Ignoring file tag."); - this.metaData.pos = this.pos; + LOGGER.warn("The file is from a different location than expected! Expected: ["+this.pos+"] but got ["+this.baseMetaData.pos+"]. Ignoring file tag."); + this.baseMetaData.pos = this.pos; } } protected void writeData(IMetaDataWriterFunc dataWriterFunc) throws IOException { - LodUtil.assertTrue(this.metaData != null); + LodUtil.assertTrue(this.baseMetaData != null); if (this.file.exists()) { validateMetaDataFile(this.file); @@ -226,10 +229,10 @@ public abstract class AbstractMetaDataContainerFile buffer.putInt(this.pos.sectionZ); buffer.putInt(checksum); buffer.put(this.pos.sectionDetailLevel); - buffer.put(this.metaData.dataLevel); - buffer.put(this.metaData.loaderVersion); - buffer.put(this.metaData.worldGenStep != null ? this.metaData.worldGenStep.value : EDhApiWorldGenerationStep.EMPTY.value); // TODO this null check shouldn't be necessary - buffer.putLong(this.metaData.dataTypeId); + buffer.put(this.baseMetaData.dataLevel); + buffer.put(this.baseMetaData.binaryDataFormatVersion); + buffer.put(this.baseMetaData.worldGenStep != null ? this.baseMetaData.worldGenStep.value : EDhApiWorldGenerationStep.EMPTY.value); // TODO this null check shouldn't be necessary + buffer.putLong(this.baseMetaData.dataTypeId); buffer.putLong(Long.MAX_VALUE); //buff.putLong(this.metaData.dataVersion.get()); // not currently implemented LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE); buffer.flip(); diff --git a/core/src/main/java/com/seibel/lod/core/file/metaData/BaseMetaData.java b/core/src/main/java/com/seibel/lod/core/file/metaData/BaseMetaData.java index 8c7498c85..105d600f8 100644 --- a/core/src/main/java/com/seibel/lod/core/file/metaData/BaseMetaData.java +++ b/core/src/main/java/com/seibel/lod/core/file/metaData/BaseMetaData.java @@ -21,11 +21,11 @@ public class BaseMetaData // Loader stuff // /** indicates what data is held in this file, this is generally a hash of the data's name */ public long dataTypeId; - public byte loaderVersion; + public byte binaryDataFormatVersion; - public BaseMetaData(DhSectionPos pos, int checksum, byte dataLevel, EDhApiWorldGenerationStep worldGenStep, long dataTypeId, byte loaderVersion) + public BaseMetaData(DhSectionPos pos, int checksum, byte dataLevel, EDhApiWorldGenerationStep worldGenStep, long dataTypeId, byte binaryDataFormatVersion) { this.pos = pos; this.checksum = checksum; @@ -34,7 +34,7 @@ public class BaseMetaData this.worldGenStep = worldGenStep; this.dataTypeId = dataTypeId; - this.loaderVersion = loaderVersion; + this.binaryDataFormatVersion = binaryDataFormatVersion; } } diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java index c5421be42..f0288cfef 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java @@ -52,7 +52,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile { super(fileHandler.computeRenderFilePath(pos), pos); this.fileHandler = fileHandler; - LodUtil.assertTrue(this.metaData == null); + LodUtil.assertTrue(this.baseMetaData == null); this.doesFileExist = this.file.exists(); } @@ -68,7 +68,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile { super(path); this.fileHandler = fileHandler; - LodUtil.assertTrue(this.metaData != null); + LodUtil.assertTrue(this.baseMetaData != null); this.doesFileExist = this.file.exists(); } @@ -163,7 +163,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile this.fileHandler.onCreateRenderFileAsync(this) .thenApply((data) -> { - this.metaData = makeMetaData(data); + this.baseMetaData = makeMetaData(data); return data; }) .thenApply((renderSource) -> this.fileHandler.onRenderFileLoaded(renderSource, this)) @@ -187,7 +187,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile { CompletableFuture.supplyAsync(() -> { - if (this.metaData == null) + if (this.baseMetaData == null) { throw new IllegalStateException("Meta data not loaded!"); } diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderSourceFileHandler.java index 40c1ee44a..ae34dca08 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderSourceFileHandler.java @@ -352,9 +352,9 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider currentRenderSource.updateFromRenderSource(newRenderSource); //file.metaData.dataVersion.set(newDataVersion); - file.metaData.dataLevel = currentRenderSource.getDataDetail(); - file.metaData.dataTypeId = RENDER_SOURCE_TYPE_ID; - file.metaData.loaderVersion = currentRenderSource.getRenderDataFormatVersion(); + file.baseMetaData.dataLevel = currentRenderSource.getDataDetail(); + file.baseMetaData.dataTypeId = RENDER_SOURCE_TYPE_ID; + file.baseMetaData.binaryDataFormatVersion = currentRenderSource.getRenderDataFormatVersion(); file.save(currentRenderSource); } @@ -363,14 +363,14 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider RenderMetaDataFile file = this.filesBySectionPos.get(renderSource.getSectionPos()); if (renderSource.isEmpty()) { - if (file == null || file.metaData == null) + if (file == null || file.baseMetaData == null) { return false; } } LodUtil.assertTrue(file != null); - LodUtil.assertTrue(file.metaData != null); + LodUtil.assertTrue(file.baseMetaData != null); // if (!this.fullDataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) // { this.updateCacheAsync(renderSource, file).join();