diff --git a/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java b/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java index 71c51072a..b63e55bc9 100644 --- a/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java +++ b/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java @@ -15,19 +15,6 @@ import java.io.IOException; import java.io.OutputStream; import java.util.concurrent.atomic.AtomicReference; -/** - * Example on how to register a loader: - * 
-     public static RenderDataSource testAndConstruct(LodDataSource dataSource, DhSectionPos sectionPos) {
-        ColumnRenderContainer container = new ColumnRenderContainer(10, -100);
-        container.startFillData(dataSource);
-        return container;
-     }
-     static {
-        RenderDataSource.registorLoader(ColumnRenderContainer::testAndConstruct, 0);
-     }
-
- */ public interface LodRenderSource { void enableRender(LodQuadTree quadTree); void disableRender(); @@ -45,5 +32,7 @@ public interface LodRenderSource { void saveRender(IClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException; - void update(DHChunkPos chunkPos, ChunkSizedData chunkData); + void update(ChunkSizedData chunkData); + + byte getRenderVersion(); } diff --git a/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java b/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java index 88c62add0..621513a8e 100644 --- a/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java +++ b/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java @@ -21,7 +21,7 @@ public class ColumnRenderLoader extends RenderSourceLoader { //TODO: Add decompressor here DataInputStream dis = new DataInputStream(data); ) { - return new ColumnRenderSource(dataFile.pos, dis, dataFile.dataVersion, level); + return new ColumnRenderSource(dataFile.pos, dis, dataFile.loaderVersion, level); } } diff --git a/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java b/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java index bc5b22b5e..ba8a3a9b3 100644 --- a/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java +++ b/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java @@ -312,7 +312,12 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype { } @Override - public void update(DHChunkPos chunkPos, ChunkSizedData chunkData) { + public void update(ChunkSizedData chunkData) { //TODO Update render data directly } + + @Override + public byte getRenderVersion() { + return LATEST_VERSION; + } } diff --git a/src/main/java/com/seibel/lod/core/a7/datatype/transform/DataRenderTransformer.java b/src/main/java/com/seibel/lod/core/a7/datatype/transform/DataRenderTransformer.java new file mode 100644 index 000000000..2a0dd94fc --- /dev/null +++ b/src/main/java/com/seibel/lod/core/a7/datatype/transform/DataRenderTransformer.java @@ -0,0 +1,30 @@ +package com.seibel.lod.core.a7.datatype.transform; + +import com.seibel.lod.core.a7.datatype.LodDataSource; +import com.seibel.lod.core.a7.datatype.LodRenderSource; +import com.seibel.lod.core.a7.datatype.column.ColumnRenderLoader; +import com.seibel.lod.core.a7.datatype.column.ColumnRenderSource; +import com.seibel.lod.core.a7.level.IClientLevel; +import com.seibel.lod.core.util.LodUtil; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +public class DataRenderTransformer { + public static final ExecutorService TRANSFORMER_THREADS + = LodUtil.makeSingleThreadPool("Data/Render Transformer"); + + public static CompletableFuture transformDataSource(LodDataSource data, IClientLevel level) { + return CompletableFuture.supplyAsync(() -> transform(data, level), TRANSFORMER_THREADS); + } + + public static CompletableFuture asyncTransformDataSource(CompletableFuture data, IClientLevel level) { + return data.thenApplyAsync((d) -> transform(d, level), TRANSFORMER_THREADS); + } + + private static LodRenderSource transform(LodDataSource dataSource, IClientLevel level) { + if (dataSource == null) return null; + return ColumnRenderLoader.loaderRegistry.get(ColumnRenderSource.class) + .stream().findFirst().get().createRender(dataSource, level); + } +} diff --git a/src/main/java/com/seibel/lod/core/a7/render/LodSection.java b/src/main/java/com/seibel/lod/core/a7/render/LodSection.java index aae566aa5..ec448c8eb 100644 --- a/src/main/java/com/seibel/lod/core/a7/render/LodSection.java +++ b/src/main/java/com/seibel/lod/core/a7/render/LodSection.java @@ -44,7 +44,7 @@ public class LodSection { public void load(IRenderSourceProvider renderDataProvider, RenderSourceLoader renderDataSourceClass) { if (loadFuture != null || lodRenderSource != null) throw new IllegalStateException("Reloading is not supported!"); - loadFuture = renderDataProvider.createRenderData(renderDataSourceClass, pos); + loadFuture = renderDataProvider.read(pos); } public void tick(LodQuadTree quadTree) { diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java b/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java index e033ef91c..e5dcdb533 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java @@ -9,14 +9,18 @@ import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; import com.seibel.lod.core.a7.datatype.DataSourceLoader; import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; +import org.apache.logging.log4j.Logger; public class MetaFile { + public static final Logger LOGGER = DhLoggerBuilder.getLogger("FileMetadata"); //Metadata format: // // 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signal the metadata format) @@ -47,9 +51,8 @@ public class MetaFile { public byte dataLevel; //Loader stuff - public DataSourceLoader loader; - public Class dataType; - public byte dataVersion; + public long dataTypeId; + public byte loaderVersion; // Load a metaFile in this path. It also automatically read the metadata. protected MetaFile(File path) throws IOException { @@ -65,22 +68,15 @@ public class MetaFile { int x = buffer.getInt(); int y = buffer.getInt(); // Unused int z = buffer.getInt(); - int checksum = buffer.getInt(); + checksum = buffer.getInt(); byte detailLevel = buffer.get(); dataLevel = buffer.get(); - byte loaderVersion = buffer.get(); + loaderVersion = buffer.get(); byte unused = buffer.get(); - long dataTypeId = buffer.getLong(); - long timestamp = buffer.getLong(); + dataTypeId = buffer.getLong(); + timestamp = buffer.getLong(); LodUtil.assertTrue(buffer.remaining() == 0); - - this.pos = new DhSectionPos(detailLevel, x, z); - this.loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion); - if (loader == null) { - throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")"); - } - this.dataType = loader.clazz; - this.dataVersion = loaderVersion; + pos = new DhSectionPos(detailLevel, x, z); } } @@ -123,16 +119,11 @@ public class MetaFile { if (!newPos.equals(pos)) { throw new IOException("Invalid file: Section position changed."); } - this.loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion); - if (loader == null) { - throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")"); - } - this.dataType = loader.clazz; - this.dataVersion = loaderVersion; + this.loaderVersion = loaderVersion; } } - protected void writeData(BiConsumer dataWriter) throws IOException { + protected void writeData(Consumer dataWriter) throws IOException { validatePath(); File tempFile = File.createTempFile("", "tmp", path.getParentFile()); tempFile.deleteOnExit(); @@ -144,10 +135,8 @@ public class MetaFile { try (OutputStream channelOut = Channels.newOutputStream(file); BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer? CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) { // TODO: Is Adler32 ok? - dataWriter.accept(this, checkedOut); + dataWriter.accept(checkedOut); checksum = (int) checkedOut.getChecksum().getValue(); - timestamp = System.currentTimeMillis(); // TODO: Do we need to use server synced time? - // Warn: This may become an attack vector! Be careful! } file.position(0); // Write metadata @@ -159,9 +148,9 @@ public class MetaFile { buff.putInt(checksum); buff.put(pos.sectionDetail); buff.put(dataLevel); - buff.put(dataVersion); + buff.put(loaderVersion); buff.put(Byte.MIN_VALUE); // Unused - buff.putLong(loader.datatypeId); + buff.putLong(dataTypeId); buff.putLong(timestamp); LodUtil.assertTrue(buff.remaining() == 0); buff.flip(); diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java b/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java index 56b66de71..c9cc635df 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java @@ -12,22 +12,21 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.DataSourceLoader; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullFormat; import com.seibel.lod.core.a7.save.io.MetaFile; import com.seibel.lod.core.a7.level.ILevel; import com.seibel.lod.core.a7.pos.DhSectionPos; -import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; -import org.apache.logging.log4j.Logger; +import org.spongepowered.asm.mixin.injection.Inject; public class DataMetaFile extends MetaFile { - public static Logger LOGGER = DhLoggerBuilder.getLogger("FileMetadata"); - private final ILevel level; + public DataSourceLoader loader; + public Class dataType; AtomicInteger localVersion = new AtomicInteger(); // This MUST be atomic // The '?' type should either be: - // Reference, or - Dirtied file that needs to be saved // SoftReference, or - Non-dirty file that can be GCed // CompletableFuture, or - File that is being loaded // null - Nothing is loaded or being loaded @@ -36,13 +35,13 @@ public class DataMetaFile extends MetaFile { //TODO: use ConcurrentAppendSingleSwapContainer instead of below: private static class GuardedMultiAppendQueue { ReentrantReadWriteLock appendLock = new ReentrantReadWriteLock(); - ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); } AtomicReference writeQueue = new AtomicReference<>(new GuardedMultiAppendQueue()); GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue(); - public void addToWriteQueue(FullFormat datatype) { + public void addToWriteQueue(ChunkSizedData datatype) { GuardedMultiAppendQueue queue = writeQueue.get(); // Using read lock is OK, because the queue's underlying data structure is thread-safe. // This lock is only used to insure on polling the queue, that the queue is not being @@ -72,6 +71,12 @@ public class DataMetaFile extends MetaFile { public DataMetaFile(ILevel level, File path) throws IOException { super(path); this.level = level; + loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion); + if (loader == null) { + throw new IOException("Invalid file: Data type loader not found: " + + dataTypeId + "(v" + loaderVersion + ")"); + } + dataType = loader.clazz; } // Make a new MetaFile. It doesn't load or write any metadata itself. @@ -81,7 +86,7 @@ public class DataMetaFile extends MetaFile { } public boolean isValid(int version) { - boolean isValid = false; + boolean isValid; // First check if write queue is empty, then check if localVersion is equal to version. // Must be done in this order as writer will increment localVersion before polling in the write queue. // Note: Be careful with the localVerion read's memory order if we do switch over to java 1.9. @@ -93,13 +98,17 @@ public class DataMetaFile extends MetaFile { isValid &= localVersion.get() == version; // The 'get()' enforce a memory barrier. return isValid; } - + + + // Suppress casting of CompletableFuture to CompletableFuture + @SuppressWarnings("unchecked") private CompletableFuture _readCached(Object obj) { // Has file cached in RAM and not freed yet. if ((obj instanceof SoftReference)) { Object inner = ((SoftReference)obj).get(); if (inner != null) { LodUtil.assertTrue(inner instanceof LodDataSource); + //TODO: Apply the write if queue is not empty return CompletableFuture.completedFuture((LodDataSource)inner); } } @@ -120,20 +129,22 @@ public class DataMetaFile extends MetaFile { CompletableFuture cached = _readCached(obj); if (cached != null) return cached; - CompletableFuture future = new CompletableFuture(); + CompletableFuture future = new CompletableFuture<>(); // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( boolean worked = data.compareAndSet(obj, future); if (!worked) return loadOrGetCached(fileReaderThreads); // Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :( - //return future.completeAsync(this::loadFile, fileReaderThreads); - CompletableFuture.supplyAsync(this::loadAndUpdateDataSource, fileReaderThreads).whenComplete((f, e) -> { + //return future.completeAsync(this::loadAndUpdateDataSource, fileReaderThreads); + CompletableFuture.supplyAsync(this::loadAndUpdateDataSource, fileReaderThreads) + .whenComplete((f, e) -> { if (e != null) { LOGGER.error("Uncaught error loading file {}: ", path, e); future.complete(null); } future.complete(f); + data.set(new SoftReference<>(f)); }); return future; } @@ -201,13 +212,27 @@ public class DataMetaFile extends MetaFile { } } + @Override + protected void updateMetaData() throws IOException { + super.updateMetaData(); + loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion); + if (loader == null) { + throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")"); + } + dataType = loader.clazz; + dataTypeId = loader.datatypeId; + } + private void write(LodDataSource data) { try { - super.writeData((meta, out) -> { - meta.dataLevel = data.getDataDetail(); - meta.dataType = data.getClass(); - meta.loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion()); - meta.dataVersion = data.getDataVersion(); + dataLevel = data.getDataDetail(); + loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion()); + dataType = data.getClass(); + dataTypeId = loader.datatypeId; + loaderVersion = data.getDataVersion(); + timestamp = System.currentTimeMillis(); // TODO: Do we need to use server synced time? + // Warn: This may become an attack vector! Be careful! + super.writeData((out) -> { try { data.saveData(level, this, out); } catch (IOException e) { diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java b/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java index 7bb45eace..61898f88e 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java @@ -1,8 +1,10 @@ package com.seibel.lod.core.a7.save.io.file; import com.seibel.lod.core.a7.datatype.LodDataSource; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullFormat; import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.objects.DHChunkPos; import java.io.File; import java.util.Collection; @@ -12,6 +14,8 @@ public interface IDataSourceProvider extends AutoCloseable { void addScannedFile(Collection detectedFiles); CompletableFuture read(DhSectionPos pos); - void write(DhSectionPos sectionPos, FullFormat chunkData); + void write(DhSectionPos sectionPos, ChunkSizedData chunkData); CompletableFuture flushAndSave(); + + boolean isCacheValid(DhSectionPos sectionPos, long timestamp); } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/file/LocalDataFileHandler.java b/src/main/java/com/seibel/lod/core/a7/save/io/file/LocalDataFileHandler.java index ef8a182e7..0ac1d7612 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/file/LocalDataFileHandler.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/file/LocalDataFileHandler.java @@ -2,10 +2,12 @@ package com.seibel.lod.core.a7.save.io.file; import com.google.common.collect.HashMultimap; import com.seibel.lod.core.a7.datatype.LodDataSource; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullFormat; import com.seibel.lod.core.a7.level.IServerLevel; import com.seibel.lod.core.a7.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.objects.DHChunkPos; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; @@ -21,15 +23,13 @@ import java.util.concurrent.ExecutorService; public class LocalDataFileHandler implements IDataSourceProvider { // Note: Single main thread only for now. May make it multi-thread later, depending on the usage. - ExecutorService fileReaderThread = LodUtil.makeSingleThreadPool("FileReaderThread"); - Logger logger = DhLoggerBuilder.getLogger("LocalDataFileHandler"); - - ConcurrentHashMap files = new ConcurrentHashMap<>(); - - boolean isScanned = false; - - File saveDir; + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + final ExecutorService fileReaderThread = LodUtil.makeSingleThreadPool("FileReaderThread"); + final ConcurrentHashMap files = new ConcurrentHashMap<>(); final IServerLevel level; + final File saveDir; + + public LocalDataFileHandler(IServerLevel level, File saveRootDir) { this.saveDir = saveRootDir; this.level = level; @@ -73,7 +73,7 @@ public class LocalDataFileHandler implements IDataSourceProvider { sb.append(fileToUse.path); sb.append("\n"); sb.append("(Other files will be renamed by appending \".old\" to their name.)"); - logger.warn(sb.toString()); + LOGGER.warn(sb.toString()); // Rename all other files with the same pos to .old for (DataMetaFile metaFile : metaFiles) { @@ -82,7 +82,7 @@ public class LocalDataFileHandler implements IDataSourceProvider { try { if (!metaFile.path.renameTo(oldFile)) throw new RuntimeException("Renaming failed"); } catch (Exception e) { - logger.error("Failed to rename file: " + metaFile.path + " to " + oldFile, e); + LOGGER.error("Failed to rename file: " + metaFile.path + " to " + oldFile, e); } } } @@ -110,7 +110,7 @@ public class LocalDataFileHandler implements IDataSourceProvider { * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ @Override - public void write(DhSectionPos sectionPos, FullFormat chunkData) { + public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { DataMetaFile metaFile = files.get(sectionPos); if (metaFile != null) { // Fast path: if there is a file for this section, just write to it. metaFile.addToWriteQueue(chunkData); @@ -142,6 +142,14 @@ public class LocalDataFileHandler implements IDataSourceProvider { return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } + @Override + public boolean isCacheValid(DhSectionPos sectionPos, long timestamp) { + DataMetaFile file = files.get(sectionPos); + if (file == null) return false; + //TODO + return true; + } + private File computeDefaultFilePath(DhSectionPos pos) { //TODO: Temp code as we haven't decided on the file naming & location yet. return new File(saveDir, pos.serialize() + ".lod"); } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java b/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java index 8d0eeb51a..f25704d27 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java @@ -1,6 +1,7 @@ package com.seibel.lod.core.a7.save.io.file; import com.seibel.lod.core.a7.datatype.LodDataSource; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullFormat; import com.seibel.lod.core.a7.pos.DhSectionPos; @@ -20,7 +21,7 @@ public class RemoteDataFileHandler implements IDataSourceProvider { } @Override - public void write(DhSectionPos sectionPos, FullFormat chunkData) { + public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { } @@ -29,6 +30,11 @@ public class RemoteDataFileHandler implements IDataSourceProvider { return null; } + @Override + public boolean isCacheValid(DhSectionPos sectionPos, long timestamp) { + return false; + } + @Override public void close() throws Exception { diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java b/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java index d8731b89d..4b109eb86 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java @@ -1,6 +1,7 @@ package com.seibel.lod.core.a7.save.io.render; -import com.seibel.lod.core.a7.datatype.full.FullFormat; +import com.seibel.lod.core.a7.datatype.LodRenderSource; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.pos.DhSectionPos; import java.io.File; @@ -8,7 +9,8 @@ import java.util.Collection; import java.util.concurrent.CompletableFuture; public interface IRenderSourceProvider extends AutoCloseable { + CompletableFuture read(DhSectionPos pos); void addScannedFile(Collection detectedFiles); - void write(DhSectionPos sectionPos, FullFormat chunkData); + void write(DhSectionPos sectionPos, ChunkSizedData chunkData); CompletableFuture flushAndSave(); } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java b/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java index 935a22fe0..e546e0698 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java @@ -1,45 +1,153 @@ package com.seibel.lod.core.a7.save.io.render; +import com.google.common.collect.HashMultimap; +import com.seibel.lod.core.a7.datatype.LodDataSource; +import com.seibel.lod.core.a7.datatype.LodRenderSource; +import com.seibel.lod.core.a7.datatype.RenderSourceLoader; +import com.seibel.lod.core.a7.datatype.column.ColumnRenderLoader; +import com.seibel.lod.core.a7.datatype.column.ColumnRenderSource; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullFormat; +import com.seibel.lod.core.a7.level.IClientLevel; +import com.seibel.lod.core.a7.save.io.file.DataMetaFile; import com.seibel.lod.core.a7.save.io.file.IDataSourceProvider; import com.seibel.lod.core.a7.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.objects.DHChunkPos; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; +import org.lwjgl.system.CallbackI; import java.io.File; -import java.util.Collection; +import java.io.IOException; +import java.util.*; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; public class RenderFileHandler implements IRenderSourceProvider { - final File renderCacheFolder; + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + final ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread"); + final ConcurrentHashMap files = new ConcurrentHashMap<>(); + final IClientLevel level; + final File saveDir; final IDataSourceProvider dataSourceProvider; - ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread"); - Logger logger = DhLoggerBuilder.getLogger("RenderCache"); - public RenderFileHandler(IDataSourceProvider sourceProvider, File renderCacheFolder) { + public RenderFileHandler(IDataSourceProvider sourceProvider, IClientLevel level, File saveRootDir) { this.dataSourceProvider = sourceProvider; - this.renderCacheFolder = renderCacheFolder; + this.level = level; + this.saveDir = saveRootDir; } + /* + * Caller must ensure that this method is called only once, + * and that this object is not used before this method is called. + */ @Override public void addScannedFile(Collection detectedFiles) { + HashMultimap filesByPos = HashMultimap.create(); + { // Sort files by pos. + for (File file : detectedFiles) { + try { + RenderMetaFile metaFile = new RenderMetaFile( + dataSourceProvider::isCacheValid, + dataSourceProvider::read, + level, file + ); + filesByPos.put(metaFile.pos, metaFile); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + // Warn for multiple files with the same pos, and then select the one with latest timestamp. + for (DhSectionPos pos : filesByPos.keySet()) { + Collection metaFiles = filesByPos.get(pos); + RenderMetaFile fileToUse; + if (metaFiles.size() > 1) { + fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.timestamp)); + { + StringBuilder sb = new StringBuilder(); + sb.append("Multiple files with the same pos: "); + sb.append(pos); + sb.append("\n"); + for (RenderMetaFile metaFile : metaFiles) { + sb.append("\t"); + sb.append(metaFile.path); + sb.append("\n"); + } + sb.append("\tUsing: "); + sb.append(fileToUse.path); + sb.append("\n"); + sb.append("(Other files will be renamed by appending \".old\" to their name.)"); + LOGGER.warn(sb.toString()); + + // Rename all other files with the same pos to .old + for (RenderMetaFile metaFile : metaFiles) { + if (metaFile == fileToUse) continue; + File oldFile = new File(metaFile.path + ".old"); + try { + if (!metaFile.path.renameTo(oldFile)) throw new RuntimeException("Renaming failed"); + } catch (Exception e) { + LOGGER.error("Failed to rename file: " + metaFile.path + " to " + oldFile, e); + } + } + } + } else { + fileToUse = metaFiles.iterator().next(); + } + // Add file to the list of files. + files.put(pos, fileToUse); + } } + /* + * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. + */ @Override - public void write(DhSectionPos sectionPos, FullFormat chunkData) { - + public CompletableFuture read(DhSectionPos pos) { + RenderMetaFile metaFile = files.computeIfAbsent(pos, (p) -> new RenderMetaFile( + dataSourceProvider::isCacheValid, + dataSourceProvider::read, + level, computeDefaultFilePath(p), p)); + return metaFile.loadOrGetCached(renderCacheThread); } + /* + * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. + */ + @Override + public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { + dataSourceProvider.write(sectionPos, chunkData); + RenderMetaFile metaFile = files.get(sectionPos); + if (metaFile != null) { // Fast path: if there is a file for this section, just write to it. + metaFile.updateChunkIfNeeded(chunkData); + } + } + + /* + * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. + */ @Override public CompletableFuture flushAndSave() { - return null; + ArrayList> futures = new ArrayList>(); + for (RenderMetaFile metaFile : files.values()) { + futures.add(metaFile.flushAndSave(renderCacheThread)); + } + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + + private File computeDefaultFilePath(DhSectionPos pos) { //TODO: Temp code as we haven't decided on the file naming & location yet. + return new File(saveDir, pos.serialize() + ".lod"); } @Override public void close() { - + ArrayList> futures = new ArrayList>(); + for (RenderMetaFile metaFile : files.values()) { + futures.add(metaFile.flushAndSave(renderCacheThread)); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } } diff --git a/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java b/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java index 195b154d7..18fad619a 100644 --- a/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java +++ b/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java @@ -1,18 +1,209 @@ package com.seibel.lod.core.a7.save.io.render; +import com.seibel.lod.core.a7.datatype.LodDataSource; +import com.seibel.lod.core.a7.datatype.LodRenderSource; +import com.seibel.lod.core.a7.datatype.RenderSourceLoader; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; +import com.seibel.lod.core.a7.datatype.full.FullFormat; +import com.seibel.lod.core.a7.datatype.transform.DataRenderTransformer; +import com.seibel.lod.core.a7.level.IClientLevel; +import com.seibel.lod.core.a7.level.ILevel; import com.seibel.lod.core.a7.save.io.MetaFile; import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.a7.save.io.file.DataMetaFile; +import com.seibel.lod.core.util.LodUtil; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.lang.ref.SoftReference; +import java.util.Optional; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class RenderMetaFile extends MetaFile { + private final IClientLevel level; + public RenderSourceLoader loader; + public Class dataType; - protected RenderMetaFile(File path) throws IOException { - super(path); + // The '?' type should either be: + // SoftReference, or - File that may still be loaded + // CompletableFuture,or - File that is being loaded + // null - Nothing is loaded or being loaded + AtomicReference data = new AtomicReference<>(null); + + //FIXME: This can cause concurrent modification of LodRenderSource. + // Not sure if it will cause issues or not. + public void updateChunkIfNeeded(ChunkSizedData chunkData) { + CompletableFuture source = _readCached(data.get()); + if (source.isDone()) source.join().update(chunkData); } - protected RenderMetaFile(File path, DhSectionPos pos) { + public CompletableFuture flushAndSave(ExecutorService renderCacheThread) { + CompletableFuture source = _readCached(data.get()); + return source.thenAccept((a)->{}); + //TODO: Should we save the data or let user re-calculate it on new load? + } + + @FunctionalInterface + public interface CacheValidator { + boolean isCacheValid(DhSectionPos sectionPos, long timestamp); + } + @FunctionalInterface + public interface CacheSourceProducer { + CompletableFuture getSourceFuture(DhSectionPos sectionPos); + } + CacheValidator validator; + CacheSourceProducer source; + + // Load a metaFile in this path. It also automatically read the metadata. + public RenderMetaFile(CacheValidator validator, CacheSourceProducer source, + IClientLevel level, File path) throws IOException { + super(path); + this.level = level; + loader = RenderSourceLoader.getLoader(dataTypeId, loaderVersion); + if (loader == null) { + throw new IOException("Invalid file: Data type loader not found: " + + dataTypeId + "(v" + loaderVersion + ")"); + } + dataType = loader.clazz; + this.validator = validator; + this.source = source; + } + + // Make a new MetaFile. It doesn't load or write any metadata itself. + public RenderMetaFile(CacheValidator validator, CacheSourceProducer source, + IClientLevel level, File path, DhSectionPos pos) { super(path, pos); + this.level = level; + this.validator = validator; + this.source = source; + } + + // Suppress casting of CompletableFuture to CompletableFuture + @SuppressWarnings("unchecked") + private CompletableFuture _readCached(Object obj) { + // Has file cached in RAM and not freed yet. + if ((obj instanceof SoftReference)) { + Object inner = ((SoftReference)obj).get(); + if (inner != null) { + LodUtil.assertTrue(inner instanceof LodRenderSource); + return CompletableFuture.completedFuture((LodRenderSource)inner); + } + } + + //==== Cached file out of scrope. ==== + // Someone is already trying to complete it. so just return the obj. + if ((obj instanceof CompletableFuture)) { + return (CompletableFuture)obj; + } + return null; + } + + // Cause: Generic Type runtime casting cannot safety check it. + // However, the Union type ensures the 'data' should only contain the listed type. + public CompletableFuture loadOrGetCached(Executor fileReaderThreads) { + Object obj = data.get(); + + CompletableFuture cached = _readCached(obj); + if (cached != null) return cached; + + // Create an empty and non-completed future. + // Note: I do this before actually filling in the future so that I can ensure only + // one task is submitted to the thread pool. + CompletableFuture future = new CompletableFuture<>(); + + // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( + boolean worked = data.compareAndSet(obj, future); + if (!worked) return loadOrGetCached(fileReaderThreads); + + // Now, there should only ever be one thread at a time here due to the CAS operation above. + + // Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :( + //return future.completeAsync(this::loadAndUpdateRenderSource, fileReaderThreads); + CompletableFuture.supplyAsync(() -> buildFuture(fileReaderThreads), fileReaderThreads) + .thenCompose((sourceCompletableFuture) -> sourceCompletableFuture) + .whenComplete((renderSource, e) -> { + if (e != null) { + LOGGER.error("Uncaught error loading file {}: ", path, e); + future.complete(null); + } + future.complete(renderSource); + data.set(new SoftReference<>(renderSource)); + }); + return future; + } + + private CompletableFuture buildFuture(Executor executorService) { + if (path.exists()) { + try { + updateMetaData(); + if (validator.isCacheValid(pos, timestamp)) { + // Load the file. + try (FileInputStream fio = getDataContent()) { + return CompletableFuture.completedFuture( + loader.loadRender(this, fio, level)); + } + } + } catch (IOException e) { + LOGGER.warn("Failed to read render cache at {}:", path, e); + LOGGER.warn("Will ignore cache file."); + } + } + // Otherwise, re-query and make the RenderSource + CompletableFuture dataFuture = source.getSourceFuture(pos); + return dataFuture.thenCombineAsync( + DataRenderTransformer.asyncTransformDataSource(dataFuture, level), + this::write, executorService); + } + + private FileInputStream getDataContent() throws IOException { + FileInputStream fin = new FileInputStream(path); + int toSkip = METADATA_SIZE; + while (toSkip > 0) { + long skipped = fin.skip(toSkip); + if (skipped == 0) { + throw new IOException("Invalid file: Failed to skip metadata."); + } + toSkip -= skipped; + } + if (toSkip != 0) { + throw new IOException("File IO Error: Failed to skip metadata."); + } + return fin; + } + + @Override + protected void updateMetaData() throws IOException { + super.updateMetaData(); + loader = RenderSourceLoader.getLoader(dataTypeId, loaderVersion); + if (loader == null) { + throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")"); + } + dataType = loader.clazz; + dataTypeId = loader.renderTypeId; + } + + private LodRenderSource write(LodDataSource parent, LodRenderSource render) { + if (parent == null) return null; + try { + //TODO: Update Timestamp & stuff based on parent + dataLevel = parent.getDataDetail(); + loader = RenderSourceLoader.getLoader(render.getClass(), render.getRenderVersion()); + dataType = render.getClass(); + dataTypeId = loader.renderTypeId; + loaderVersion = render.getRenderVersion(); + super.writeData((out) -> { + try { + render.saveRender(level, this, out); + } catch (IOException e) { + LOGGER.error("Failed to save data for file {}", path, e); + } + }); + } catch (IOException e) { + LOGGER.error("Failed to write data for file {}", path, e); + } + return render; } }