diff --git a/core/src/main/java/com/seibel/lod/core/datatype/AbstractRenderSourceLoader.java b/core/src/main/java/com/seibel/lod/core/datatype/AbstractRenderSourceLoader.java index fa0e8a6ef..940f7f75a 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/AbstractRenderSourceLoader.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/AbstractRenderSourceLoader.java @@ -3,7 +3,7 @@ package com.seibel.lod.core.datatype; import com.google.common.collect.HashMultimap; import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.level.IDhLevel; -import com.seibel.lod.core.file.renderfile.RenderMetaFile; +import com.seibel.lod.core.file.renderfile.RenderMetaDataFile; import java.io.IOException; import java.io.InputStream; @@ -64,7 +64,7 @@ public abstract class AbstractRenderSourceLoader } /** Can return null if the file is out of date or something */ - public abstract ILodRenderSource loadRender(RenderMetaFile renderFile, InputStream data, IDhLevel level) throws IOException; + public abstract ILodRenderSource loadRender(RenderMetaDataFile renderFile, InputStream data, IDhLevel level) throws IOException; public abstract ILodRenderSource createRender(ILodDataSource dataSource, IDhClientLevel level); diff --git a/core/src/main/java/com/seibel/lod/core/datatype/ILodRenderSource.java b/core/src/main/java/com/seibel/lod/core/datatype/ILodRenderSource.java index dcb6affa3..9beed3f1a 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/ILodRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/ILodRenderSource.java @@ -5,7 +5,7 @@ import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.render.LodQuadTree; import com.seibel.lod.core.render.RenderBuffer; -import com.seibel.lod.core.file.renderfile.RenderMetaFile; +import com.seibel.lod.core.file.renderfile.RenderMetaDataFile; import java.io.IOException; import java.io.OutputStream; @@ -32,7 +32,7 @@ public interface ILodRenderSource */ boolean trySwapRenderBuffer(LodQuadTree quadTree, AtomicReference referenceSlot); - void saveRender(IDhClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException; + void saveRender(IDhClientLevel level, RenderMetaDataFile file, OutputStream dataStream) throws IOException; byte getRenderVersion(); diff --git a/core/src/main/java/com/seibel/lod/core/datatype/PlaceHolderRenderSource.java b/core/src/main/java/com/seibel/lod/core/datatype/PlaceHolderRenderSource.java index 88896c2a5..74bc380f4 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/PlaceHolderRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/PlaceHolderRenderSource.java @@ -5,7 +5,7 @@ import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.render.LodQuadTree; import com.seibel.lod.core.render.RenderBuffer; -import com.seibel.lod.core.file.renderfile.RenderMetaFile; +import com.seibel.lod.core.file.renderfile.RenderMetaDataFile; import java.io.IOException; import java.io.OutputStream; @@ -37,7 +37,7 @@ public class PlaceHolderRenderSource implements ILodRenderSource public boolean trySwapRenderBuffer(LodQuadTree quadTree, AtomicReference referenceSlots) { return false; } @Override - public void saveRender(IDhClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException + public void saveRender(IDhClientLevel level, RenderMetaDataFile file, OutputStream dataStream) throws IOException { throw new UnsupportedOperationException("EmptyRenderSource should NEVER be saved!"); } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java index 9d6b13a31..119bc1d77 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java @@ -8,7 +8,7 @@ import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.datatype.ILodRenderSource; import com.seibel.lod.core.datatype.AbstractRenderSourceLoader; import com.seibel.lod.core.level.IDhLevel; -import com.seibel.lod.core.file.renderfile.RenderMetaFile; +import com.seibel.lod.core.file.renderfile.RenderMetaDataFile; import com.seibel.lod.core.util.LodUtil; import java.io.DataInputStream; @@ -22,7 +22,7 @@ public class ColumnRenderLoader extends AbstractRenderSourceLoader } @Override - public ILodRenderSource loadRender(RenderMetaFile dataFile, InputStream data, IDhLevel level) throws IOException { + public ILodRenderSource loadRender(RenderMetaDataFile dataFile, InputStream data, IDhLevel level) throws IOException { DataInputStream dis = new DataInputStream(data); // DO NOT CLOSE return new ColumnRenderSource(dataFile.pos, dis, dataFile.metaData.loaderVersion, level); } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java index 24eb191f2..f6017041a 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java @@ -7,7 +7,7 @@ import com.seibel.lod.core.datatype.transform.FullToColumnTransformer; import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.render.RenderBuffer; -import com.seibel.lod.core.file.renderfile.RenderMetaFile; +import com.seibel.lod.core.file.renderfile.RenderMetaDataFile; import com.seibel.lod.core.enums.ELodDirection; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.level.IDhLevel; @@ -450,7 +450,7 @@ public class ColumnRenderSource implements ILodRenderSource, IColumnDatatype } @Override - public void saveRender(IDhClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException + public void saveRender(IDhClientLevel level, RenderMetaDataFile file, OutputStream dataStream) throws IOException { DataOutputStream dos = new DataOutputStream(dataStream); // DO NOT CLOSE writeData(dos); diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java index 783a45951..090f9fa5f 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java @@ -7,7 +7,7 @@ import com.seibel.lod.core.datatype.full.ChunkSizedData; import com.seibel.lod.core.datatype.full.FullDataSource; import com.seibel.lod.core.datatype.full.SparseDataSource; import com.seibel.lod.core.datatype.full.SpottyDataSource; -import com.seibel.lod.core.file.subDimMatching.MetaFile; +import com.seibel.lod.core.file.metaData.MetaData; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.pos.DhLodPos; import com.seibel.lod.core.pos.DhSectionPos; @@ -316,7 +316,7 @@ public class DataFileHandler implements IDataSourceProvider { } @Override - public ILodDataSource onDataFileLoaded(ILodDataSource source, MetaFile.MetaData metaData, + public ILodDataSource onDataFileLoaded(ILodDataSource source, MetaData metaData, Consumer onUpdated, Function updater) { boolean changed = updater.apply(source); if (changed) metaData.dataVersion.incrementAndGet(); @@ -329,7 +329,7 @@ public class DataFileHandler implements IDataSourceProvider { return source; } @Override - public CompletableFuture onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function updater, Consumer onUpdated) { + public CompletableFuture onDataFileRefresh(ILodDataSource source, MetaData metaData, Function updater, Consumer onUpdated) { return CompletableFuture.supplyAsync(() -> { ILodDataSource sourceLocal = source; boolean changed = updater.apply(sourceLocal); diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java index 971c1e468..f50021201 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java @@ -11,8 +11,9 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import com.seibel.lod.core.datatype.ILodDataSource; import com.seibel.lod.core.datatype.AbstractDataSourceLoader; import com.seibel.lod.core.datatype.full.ChunkSizedData; +import com.seibel.lod.core.file.metaData.MetaData; import com.seibel.lod.core.pos.DhLodPos; -import com.seibel.lod.core.file.subDimMatching.MetaFile; +import com.seibel.lod.core.file.metaData.MetaDataFile; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; @@ -20,7 +21,7 @@ import com.seibel.lod.core.util.AtomicsUtil; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; -public class DataMetaFile extends MetaFile +public class DataMetaFile extends MetaDataFile { private static final Logger LOGGER = DhLoggerBuilder.getLogger(DataMetaFile.class.getSimpleName()); diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java b/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java index 2402a7775..ddcb2f3fa 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java @@ -2,7 +2,7 @@ package com.seibel.lod.core.file.datafile; import com.seibel.lod.core.datatype.ILodDataSource; import com.seibel.lod.core.datatype.full.ChunkSizedData; -import com.seibel.lod.core.file.subDimMatching.MetaFile; +import com.seibel.lod.core.file.metaData.MetaData; import com.seibel.lod.core.pos.DhSectionPos; import java.io.File; @@ -23,8 +23,8 @@ public interface IDataSourceProvider extends AutoCloseable { boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion); CompletableFuture onCreateDataFile(DataMetaFile file); - ILodDataSource onDataFileLoaded(ILodDataSource source, MetaFile.MetaData metaData, Consumer onUpdated, Function updater); - CompletableFuture onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function updater, Consumer onUpdated); + ILodDataSource onDataFileLoaded(ILodDataSource source, MetaData metaData, Consumer onUpdated, Function updater); + CompletableFuture onDataFileRefresh(ILodDataSource source, MetaData metaData, Function updater, Consumer onUpdated); File computeDataFilePath(DhSectionPos pos); Executor getIOExecutor(); diff --git a/core/src/main/java/com/seibel/lod/core/file/metaData/MetaData.java b/core/src/main/java/com/seibel/lod/core/file/metaData/MetaData.java new file mode 100644 index 000000000..4772f715c --- /dev/null +++ b/core/src/main/java/com/seibel/lod/core/file/metaData/MetaData.java @@ -0,0 +1,44 @@ +package com.seibel.lod.core.file.metaData; + +import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.pos.DhSectionPos; +import com.seibel.lod.core.util.LodUtil; +import com.seibel.lod.core.util.objects.UnclosableOutputStream; +import org.apache.logging.log4j.Logger; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.Adler32; +import java.util.zip.CheckedOutputStream; + +/** + * See {@link MetaDataFile} for a byte map inorder to see the currently used bytes + */ +public class MetaData +{ + public DhSectionPos pos; + public int checksum; + public AtomicLong dataVersion; + public byte dataLevel; + //Loader stuff + public long dataTypeId; + public byte loaderVersion; + + public MetaData(DhSectionPos pos, int checksum, long dataVersion, byte dataLevel, long dataTypeId, byte loaderVersion) + { + this.pos = pos; + this.checksum = checksum; + this.dataVersion = new AtomicLong(dataVersion); + this.dataLevel = dataLevel; + this.dataTypeId = dataTypeId; + this.loaderVersion = loaderVersion; + } + +} diff --git a/core/src/main/java/com/seibel/lod/core/file/metaData/MetaDataFile.java b/core/src/main/java/com/seibel/lod/core/file/metaData/MetaDataFile.java new file mode 100644 index 000000000..7cff32d1d --- /dev/null +++ b/core/src/main/java/com/seibel/lod/core/file/metaData/MetaDataFile.java @@ -0,0 +1,245 @@ +package com.seibel.lod.core.file.metaData; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.Adler32; +import java.util.zip.CheckedOutputStream; + +import com.seibel.lod.core.pos.DhSectionPos; +import com.seibel.lod.core.util.objects.UnclosableOutputStream; +import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.util.LodUtil; +import org.apache.logging.log4j.Logger; + +/** + * Used size: 40 bytes
+ * Remaining space: 24 bytes
+ * Total size: 64 bytes


+ * + * + * Metadata format:

+ * + * 4 bytes: metadata identifier bytes: "DHv0" (in ascii: 0x44 48 76 30) this signals the file is in the metadata format
+ * 4 bytes: section X position
+ * 4 bytes: section Y position (Unused, for future proofing)
+ * 4 bytes: section Z position

+ * + * 4 bytes: data checksum
//TODO: Implement checksum + * 1 byte: section detail level
+ * 1 byte: data detail level // Note: not sure if this is needed
+ * 1 byte: loader version
+ * 1 byte: unused

+ * + * 8 bytes: datatype identifier

+ * + * 8 bytes: data version + */ +public abstract class MetaDataFile +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + public static final int METADATA_SIZE = 64; + public static final int METADATA_RESERVED_SIZE = 24; + /** equivalent to "DHv0" */ + public static final int METADATA_IDENTITY_BYTES = 0x44_48_76_30; + + /** + * Currently set to false because for some reason + * Window is throwing PermissionDeniedException when trying to atomic replace a file... + */ + public static final boolean USE_ATOMIC_MOVE_REPLACE = false; + + + public volatile MetaData metaData = null; + /** also defined in {@link MetaDataFile#metaData} */ + public final DhSectionPos pos; + + public File path; + + + + //==============// + // constructors // + //==============// + + /** Create a metaFile in this path. If the path has a file, throws FileAlreadyExistsException */ + protected MetaDataFile(File path, DhSectionPos pos) throws IOException + { + this.path = path; + this.pos = pos; + if (path.exists()) + { + throw new FileAlreadyExistsException(path.toString()); + } + } + + /** + * Creates a {@link MetaDataFile} with the file at the given path. + * @throws IOException if the file was formatted incorrectly + * @throws FileNotFoundException if no file exists for the given path + */ + protected MetaDataFile(File path) throws IOException, FileNotFoundException + { + this.path = path; + if (!path.exists()) + { + throw new FileNotFoundException("File not found at [" + path + "]"); + } + + validateMetaDataFile(this.path); + this.metaData = readMetaDataFromFile(path); + this.pos = this.metaData.pos; + } + /** + * Attempts to create a new {@link MetaDataFile} from the given file. + * @throws IOException if the file was formatted incorrectly + */ + private static MetaData readMetaDataFromFile(File file) throws IOException + { + try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) + { + ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE); + channel.read(buffer, 0); + channel.close(); + buffer.flip(); + + int idBytes = buffer.getInt(); + if (idBytes != METADATA_IDENTITY_BYTES) + { + throw new IOException("Invalid file format: Metadata Identity byte check failed. Expected: [" + METADATA_IDENTITY_BYTES + "], Actual: [" + idBytes + "]."); + } + + int x = buffer.getInt(); + int y = buffer.getInt(); // Unused + int z = buffer.getInt(); + int checksum = buffer.getInt(); + byte detailLevel = buffer.get(); + byte dataLevel = buffer.get(); + byte loaderVersion = buffer.get(); + byte unused = buffer.get(); + long dataTypeId = buffer.getLong(); + long timestamp = buffer.getLong(); + LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE); + DhSectionPos dataPos = new DhSectionPos(detailLevel, x, z); + + return new MetaData(dataPos, checksum, timestamp, dataLevel, dataTypeId, loaderVersion); + } + } + + + + //================// + // helper methods // + //================// + + /** Throws an {@link IOException} if the given file isn't valid */ + private static void validateMetaDataFile(File file) throws IOException + { + if (!file.exists()) throw new IOException("File missing"); + if (!file.isFile()) throw new IOException("Not a file"); + if (!file.canRead()) throw new IOException("File not readable"); + if (!file.canWrite()) throw new IOException("File not writable"); + } + + /** Sets this object's {@link MetaDataFile#metaData} using the set {@link MetaDataFile#path} */ + protected void loadMetaData() throws IOException + { + validateMetaDataFile(this.path); + this.metaData = readMetaDataFromFile(this.path); + if (!this.metaData.pos.equals(this.pos)) + { + LOGGER.warn("The file is from a different location than expected! Expected: [{}] but got [{}]. Ignoring file tag.", this.pos, this.metaData.pos); + this.metaData.pos = this.pos; + } + } + + protected void writeData(IMetaDataWriter dataWriter) throws IOException + { + LodUtil.assertTrue(this.metaData != null); + if (this.path.exists()) + { + validateMetaDataFile(this.path); + } + + File writerFile; + if (USE_ATOMIC_MOVE_REPLACE) + { + writerFile = new File(this.path.getPath() + ".tmp"); + writerFile.deleteOnExit(); + } + else + { + writerFile = this.path; + } + + try (FileChannel file = FileChannel.open(writerFile.toPath(), + StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) + { + { + file.position(METADATA_SIZE); + int checksum; + try (OutputStream channelOut = new UnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel + BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer? + CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) + { // TODO: Is Adler32 ok? + dataWriter.writeBufferToFile(checkedOut); + checksum = (int) checkedOut.getChecksum().getValue(); + } + file.position(0); + // Write metadata + ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE); + buff.putInt(METADATA_IDENTITY_BYTES); + buff.putInt(this.pos.sectionX); + buff.putInt(Integer.MIN_VALUE); // Unused + buff.putInt(this.pos.sectionZ); + buff.putInt(checksum); + buff.put(this.pos.sectionDetail); + buff.put(this.metaData.dataLevel); + buff.put(this.metaData.loaderVersion); + buff.put(Byte.MIN_VALUE); // Unused + buff.putLong(this.metaData.dataTypeId); + buff.putLong(this.metaData.dataVersion.get()); + LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE); + buff.flip(); + file.write(buff); + } + file.close(); + if (USE_ATOMIC_MOVE_REPLACE) + { + // Atomic move / replace the actual file + Files.move(writerFile.toPath(), this.path.toPath(), StandardCopyOption.ATOMIC_MOVE); + } + } + finally + { + try + { + if (USE_ATOMIC_MOVE_REPLACE && writerFile.exists()) + { + boolean fileRemoved = writerFile.delete(); // Delete temp file. Ignore errors if it fails. + } + } + catch (SecurityException ignored) { } + } + } + + + + //================// + // helper classes // + //================// + + @FunctionalInterface + public interface IMetaDataWriter + { + void writeBufferToFile(T t) throws IOException; + } + +} diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/IRenderSourceProvider.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/IRenderSourceProvider.java index d478f2b58..08eb3f0a7 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/IRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/IRenderSourceProvider.java @@ -8,10 +8,14 @@ import java.io.File; import java.util.Collection; import java.util.concurrent.CompletableFuture; -public interface IRenderSourceProvider extends AutoCloseable { +public interface IRenderSourceProvider extends AutoCloseable +{ CompletableFuture read(DhSectionPos pos); void addScannedFile(Collection detectedFiles); void write(DhSectionPos sectionPos, ChunkSizedData chunkData); CompletableFuture flushAndSave(); + + /** Returns true if the data was refreshed, false otherwise */ boolean refreshRenderSource(ILodRenderSource source); + } diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java index 480378ff5..4a30163bf 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java @@ -26,220 +26,305 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -public class RenderFileHandler implements IRenderSourceProvider { +public class RenderFileHandler implements IRenderSourceProvider +{ private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - final ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread"); - final ConcurrentHashMap files = new ConcurrentHashMap<>(); - final IDhClientLevel level; - final File saveDir; - final IDataSourceProvider dataSourceProvider; - - public RenderFileHandler(IDataSourceProvider sourceProvider, IDhClientLevel level, File saveRootDir) { + + private final ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread"); + private final ConcurrentHashMap files = new ConcurrentHashMap<>(); + private final IDhClientLevel level; + private final File saveDir; + private final IDataSourceProvider dataSourceProvider; + + + + public RenderFileHandler(IDataSourceProvider sourceProvider, IDhClientLevel level, File saveRootDir) + { this.dataSourceProvider = sourceProvider; this.level = level; this.saveDir = saveRootDir; } - - /* + + + + /** * Caller must ensure that this method is called only once, - * and that this object is not used before this method is called. + * and that the given files are not used before this method is called. */ @Override - public void addScannedFile(Collection detectedFiles) { - HashMultimap filesByPos = HashMultimap.create(); - { // Sort files by pos. - for (File file : detectedFiles) { - try { - RenderMetaFile metaFile = new RenderMetaFile(this, file); - filesByPos.put(metaFile.pos, metaFile); - } catch (IOException e) { - LOGGER.error("Failed to read render meta file at {}: ", file, e); - File corruptedFile = new File(file.getParentFile(), file.getName() + ".corrupted"); - if (corruptedFile.exists()) corruptedFile.delete(); - if (file.renameTo(corruptedFile)) { - LOGGER.error("Renamed corrupted file to {}", file.getName() + ".corrupted"); - } else { - LOGGER.error("Failed to rename corrupted file to {}. Will try and delete file", file.getName() + ".corrupted"); - file.delete(); - } - } - } - } - - // Warn for multiple files with the same pos, and then select the one with the latest timestamp. - for (DhSectionPos pos : filesByPos.keySet()) { - Collection metaFiles = filesByPos.get(pos); - RenderMetaFile fileToUse; - if (metaFiles.size() > 1) { - fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get())); - { - StringBuilder sb = new StringBuilder(); - sb.append("Multiple files with the same pos: "); - sb.append(pos); - sb.append("\n"); - for (RenderMetaFile metaFile : metaFiles) { - sb.append("\t"); - sb.append(metaFile.path); - sb.append("\n"); - } - sb.append("\tUsing: "); - sb.append(fileToUse.path); - sb.append("\n"); - sb.append("(Other files will be renamed by appending \".old\" to their name.)"); - LOGGER.warn(sb.toString()); - - // Rename all other files with the same pos to .old - for (RenderMetaFile metaFile : metaFiles) { - if (metaFile == fileToUse) continue; - File oldFile = new File(metaFile.path + ".old"); - try { - if (!metaFile.path.renameTo(oldFile)) throw new RuntimeException("Renaming failed"); - } catch (Exception e) { - LOGGER.error("Failed to rename file: " + metaFile.path + " to " + oldFile, e); - } - } - } - } else { - fileToUse = metaFiles.iterator().next(); - } - // Add file to the list of files. - files.put(pos, fileToUse); - } - } - - /* - * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. - */ + public void addScannedFile(Collection newRenderFiles) + { + HashMultimap filesByPos = HashMultimap.create(); + + // Sort files by pos. + for (File file : newRenderFiles) + { + try + { + RenderMetaDataFile metaFile = new RenderMetaDataFile(this, file); + filesByPos.put(metaFile.pos, metaFile); + } + catch (IOException e) + { + LOGGER.error("Failed to read render meta file at [{}]. Error: ", file, e); + String corruptedFileName = file.getName() + ".corrupted"; + + File corruptedFile = new File(file.getParentFile(), corruptedFileName); + if (corruptedFile.exists()) + { + // could happen if there was a corrupted file before that was removed + corruptedFile.delete(); + } + + + if (file.renameTo(corruptedFile)) + { + LOGGER.error("Renamed corrupted file to [{}].", file.getName() + ".corrupted"); + } + else + { + LOGGER.error("Failed to rename corrupted file to [{}]. Attempting to delete file...", corruptedFileName); + if (!file.delete()) + { + LOGGER.error("Unable to delete corrupted file [{}].", corruptedFileName); + } + } + } + } + + + + // Warn for multiple files with the same pos, and then select the one with the latest timestamp. + for (DhSectionPos pos : filesByPos.keySet()) + { + Collection metaFiles = filesByPos.get(pos); + RenderMetaDataFile fileToUse; + if (metaFiles.size() > 1) + { + fileToUse = Collections.max(metaFiles, Comparator.comparingLong(renderMetaDataFile -> + renderMetaDataFile.metaData.dataVersion.get())); + { + StringBuilder sb = new StringBuilder(); + sb.append("Multiple files with the same pos: "); + sb.append(pos); + sb.append("\n"); + for (RenderMetaDataFile metaFile : metaFiles) + { + sb.append("\t"); + sb.append(metaFile.path); + sb.append("\n"); + } + sb.append("\tUsing: "); + sb.append(fileToUse.path); + sb.append("\n"); + sb.append("(Other files will be renamed by appending \".old\" to their name.)"); + LOGGER.warn(sb.toString()); + + // Rename all other files with the same pos to .old + for (RenderMetaDataFile metaFile : metaFiles) + { + if (metaFile == fileToUse) + continue; + File oldFile = new File(metaFile.path + ".old"); + try + { + if (!metaFile.path.renameTo(oldFile)) + throw new RuntimeException("Renaming failed"); + } + catch (Exception e) + { + LOGGER.error("Failed to rename file: [" + metaFile.path + "] to [" + oldFile + "]", e); + } + } + } + } + else + { + fileToUse = metaFiles.iterator().next(); + } + + // Add this file to the list of files. + this.files.put(pos, fileToUse); + } + } + + /** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ @Override - public CompletableFuture read(DhSectionPos pos) { - RenderMetaFile metaFile = files.get(pos); - if (metaFile == null) { - RenderMetaFile newMetaFile; - try { - newMetaFile = new RenderMetaFile(this, pos); - } catch (IOException e) { - LOGGER.error("IOException on creating new render file at {}", pos, e); - return null; - } - metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. - if (metaFile == null) metaFile = newMetaFile; - } - return metaFile.loadOrGetCached(renderCacheThread, level).handle( - (render, e) -> { - if (e != null) { - LOGGER.error("Uncaught error on {}:", pos, e); - } - if (render != null) return render; - return new PlaceHolderRenderSource(pos); - } + public CompletableFuture read(DhSectionPos pos) + { + RenderMetaDataFile metaFile = this.files.get(pos); + if (metaFile == null) + { + RenderMetaDataFile newMetaFile; + try + { + newMetaFile = new RenderMetaDataFile(this, pos); + } + catch (IOException e) + { + LOGGER.error("IOException on creating new render file at {}", pos, e); + return null; + } + + metaFile = this.files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. + if (metaFile == null) + { + metaFile = newMetaFile; + } + } + + return metaFile.loadOrGetCached(this.renderCacheThread, this.level).handle( + (renderSource, exception) -> + { + if (exception != null) + { + LOGGER.error("Uncaught error on {}:", pos, exception); + } + + return (renderSource != null) ? renderSource : new PlaceHolderRenderSource(pos); + } ); } - - /* - * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. - */ + + /* This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ @Override - public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { - if (chunkData.getBBoxLodPos().convertUpwardsTo((byte)6).equals(new DhLodPos((byte)6, 10, -11))) { + public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) + { + // can be used for debugging + if (chunkData.getBBoxLodPos().convertUpwardsTo((byte)6).equals(new DhLodPos((byte)6, 10, -11))) + { int doNothing = 0; } - recursive_write(sectionPos,chunkData); - dataSourceProvider.write(sectionPos, chunkData); + this.writeRecursively(sectionPos,chunkData); + this.dataSourceProvider.write(sectionPos, chunkData); } - - private void recursive_write(DhSectionPos sectPos, ChunkSizedData chunkData) { - if (!sectPos.getSectionBBoxPos().overlaps(new DhLodPos((byte) (4 + chunkData.dataDetail), chunkData.x, chunkData.z))) return; - if (sectPos.sectionDetail > ColumnRenderSource.SECTION_SIZE_OFFSET) { - recursive_write(sectPos.getChildByIndex(0), chunkData); - recursive_write(sectPos.getChildByIndex(1), chunkData); - recursive_write(sectPos.getChildByIndex(2), chunkData); - recursive_write(sectPos.getChildByIndex(3), chunkData); - } - RenderMetaFile metaFile = files.get(sectPos); - if (metaFile != null) { // Fast path: if there is a file for this section, just write to it. - metaFile.updateChunkIfNeeded(chunkData, level); - } - } - - /* - * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. - */ + + private void writeRecursively(DhSectionPos sectPos, ChunkSizedData chunkData) + { + if (!sectPos.getSectionBBoxPos().overlaps(new DhLodPos((byte) (4 + chunkData.dataDetail), chunkData.x, chunkData.z))) + { + return; + } + + + if (sectPos.sectionDetail > ColumnRenderSource.SECTION_SIZE_OFFSET) + { + this.writeRecursively(sectPos.getChildByIndex(0), chunkData); + this.writeRecursively(sectPos.getChildByIndex(1), chunkData); + this.writeRecursively(sectPos.getChildByIndex(2), chunkData); + this.writeRecursively(sectPos.getChildByIndex(3), chunkData); + } + + RenderMetaDataFile metaFile = this.files.get(sectPos); + // Fast path: if there is a file for this section, just write to it. + if (metaFile != null) + { + metaFile.updateChunkIfNeeded(chunkData, this.level); + } + } + + /** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ @Override - public CompletableFuture flushAndSave() { - ArrayList> futures = new ArrayList>(); - for (RenderMetaFile metaFile : files.values()) { - futures.add(metaFile.flushAndSave(renderCacheThread)); - } - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); - } - - private File computeDefaultFilePath(DhSectionPos pos) { //TODO: Temp code as we haven't decided on the file naming & location yet. - return new File(saveDir, pos.serialize() + ".lod"); + public CompletableFuture flushAndSave() + { + ArrayList> futures = new ArrayList<>(); + for (RenderMetaDataFile metaFile : this.files.values()) + { + futures.add(metaFile.flushAndSave(this.renderCacheThread)); + } + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + + private File computeDefaultFilePath(DhSectionPos pos) + { + //TODO: Temp code as we haven't decided on the file naming & location yet. + return new File(this.saveDir, pos.serialize() + ".lod"); } @Override - public void close() { - ArrayList> futures = new ArrayList>(); - for (RenderMetaFile metaFile : files.values()) { - futures.add(metaFile.flushAndSave(renderCacheThread)); + public void close() + { + ArrayList> futures = new ArrayList<>(); + for (RenderMetaDataFile metaFile : this.files.values()) + { + futures.add(metaFile.flushAndSave(this.renderCacheThread)); } CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } - public File computeRenderFilePath(DhSectionPos pos) { - return new File(saveDir, pos.serialize() + ".lod"); + public File computeRenderFilePath(DhSectionPos pos) + { + return new File(this.saveDir, pos.serialize() + ".lod"); } - public CompletableFuture onCreateRenderFile(RenderMetaFile file) { - final int vertSize = Config.Client.Graphics.Quality.verticalQuality - .get().calculateMaxVerticalData((byte) (file.pos.sectionDetail - ColumnRenderSource.SECTION_SIZE_OFFSET)); - return CompletableFuture.completedFuture( - new ColumnRenderSource(file.pos, vertSize, level.getMinY())); - } + public CompletableFuture onCreateRenderFile(RenderMetaDataFile file) + { + final int vertSize = Config.Client.Graphics.Quality.verticalQuality + .get().calculateMaxVerticalData((byte) (file.pos.sectionDetail - ColumnRenderSource.SECTION_SIZE_OFFSET)); + return CompletableFuture.completedFuture( + new ColumnRenderSource(file.pos, vertSize, this.level.getMinY())); + } private final ConcurrentHashMap cacheRecreationGuards = new ConcurrentHashMap<>(); - private void updateCache(ILodRenderSource data, RenderMetaFile file) { - if (cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null) return; - final WeakReference dataRef = new WeakReference<>(data); - CompletableFuture dataFuture = dataSourceProvider.read(data.getSectionPos()); - dataFuture = dataFuture.thenApply((d) -> { - if (dataRef.get() == null) throw new UncheckedInterruptedException(); - LodUtil.assertTrue(d != null); - return d; - }).exceptionally((ex) -> { - if (ex != null) - LOGGER.error("Uncaught exception when getting data for updateCache()", ex); - return null; - }); + private void updateCache(ILodRenderSource data, RenderMetaDataFile file) + { + if (this.cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null) + { + return; + } + + final WeakReference dataRef = new WeakReference<>(data); + CompletableFuture dataFuture = this.dataSourceProvider.read(data.getSectionPos()); + dataFuture = dataFuture.thenApply((dataSource) -> + { + if (dataRef.get() == null) + throw new UncheckedInterruptedException(); + LodUtil.assertTrue(dataSource != null); + return dataSource; + }).exceptionally((ex) -> + { + if (ex != null) + { + LOGGER.error("Uncaught exception when getting data for updateCache()", ex); + } + + return null; + }); + + LOGGER.info("Recreating cache for {}", data.getSectionPos()); + DataRenderTransformer.asyncTransformDataSource(dataFuture, this.level) + .thenAccept((newRenderDataSource) -> this.write(dataRef.get(), file, newRenderDataSource, this.dataSourceProvider.getCacheVersion(data.getSectionPos()))) + .exceptionally((ex) -> { + if (!UncheckedInterruptedException.isThrowableInterruption(ex)) + LOGGER.error("Exception when updating render file using data source: ", ex); + return null; + }).thenRun(() -> this.cacheRecreationGuards.remove(file.pos)); + } - LOGGER.info("Recreating cache for {}", data.getSectionPos()); - DataRenderTransformer.asyncTransformDataSource(dataFuture , level) - .thenAccept((newData) -> write(dataRef.get(), file, newData, dataSourceProvider.getCacheVersion(data.getSectionPos()))) - .exceptionally((ex) -> { - if (!UncheckedInterruptedException.isThrowableInterruption(ex)) - LOGGER.error("Exception when updating render file using data source: ", ex); - return null; - }).thenRun(() -> cacheRecreationGuards.remove(file.pos)); - - } - - public ILodRenderSource onRenderFileLoaded(ILodRenderSource data, RenderMetaFile file) { - if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) { - updateCache(data, file); + public ILodRenderSource onRenderFileLoaded(ILodRenderSource data, RenderMetaDataFile file) + { + if (!this.dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) + { + this.updateCache(data, file); } return data; } - - public ILodRenderSource onLoadingRenderFile(RenderMetaFile file) { - return null; //Default behaviour + + public ILodRenderSource onLoadingRenderFile(RenderMetaDataFile file) + { + return null; // Default behavior: do nothing } - - private void write(ILodRenderSource target, RenderMetaFile file, - ILodRenderSource newData, long newDataVersion) { - if (target == null) return; - if (newData == null) return; + + private void write(ILodRenderSource target, RenderMetaDataFile file, + ILodRenderSource newData, long newDataVersion) + { + if (target == null || newData == null) + { + return; + } + target.updateFromRenderSource(newData); file.metaData.dataVersion.set(newDataVersion); file.metaData.dataLevel = target.getDataDetail(); @@ -247,29 +332,37 @@ public class RenderFileHandler implements IRenderSourceProvider { file.dataType = target.getClass(); file.metaData.dataTypeId = file.loader.renderTypeId; file.metaData.loaderVersion = target.getRenderVersion(); - file.save(target, level); + file.save(target, this.level); } - - public void onReadRenderSourceFromCache(RenderMetaFile file, ILodRenderSource data) { - if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) { - updateCache(data, file); + + public void onReadRenderSourceFromCache(RenderMetaDataFile file, ILodRenderSource data) + { + if (!this.dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) + { + this.updateCache(data, file); } } - - public boolean refreshRenderSource(ILodRenderSource source) { - RenderMetaFile file = files.get(source.getSectionPos()); - if (source instanceof PlaceHolderRenderSource) { - if (file == null || file.metaData == null) { + + public boolean refreshRenderSource(ILodRenderSource source) + { + RenderMetaDataFile file = this.files.get(source.getSectionPos()); + if (source instanceof PlaceHolderRenderSource) + { + if (file == null || file.metaData == null) + { return false; } } + LodUtil.assertTrue(file != null); LodUtil.assertTrue(file.metaData != null); - if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) { - updateCache(source, file); + if (!this.dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) + { + this.updateCache(source, file); return true; } + return false; } - + } diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java new file mode 100644 index 000000000..b5e12cbca --- /dev/null +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java @@ -0,0 +1,289 @@ +package com.seibel.lod.core.file.renderfile; + +import com.seibel.lod.core.datatype.ILodRenderSource; +import com.seibel.lod.core.datatype.AbstractRenderSourceLoader; +import com.seibel.lod.core.datatype.full.ChunkSizedData; +import com.seibel.lod.core.file.metaData.MetaData; +import com.seibel.lod.core.level.IDhClientLevel; +import com.seibel.lod.core.level.IDhLevel; +import com.seibel.lod.core.pos.DhLodPos; +import com.seibel.lod.core.file.metaData.MetaDataFile; +import com.seibel.lod.core.pos.DhSectionPos; +import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.util.LodUtil; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.lang.ref.SoftReference; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; + +public class RenderMetaDataFile extends MetaDataFile +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + public AbstractRenderSourceLoader loader; + public Class dataType; + + // The '?' type should either be: + // SoftReference, or - File that may still be loaded + // CompletableFuture,or - File that is being loaded + // null - Nothing is loaded or being loaded + AtomicReference data = new AtomicReference<>(null); + +// @FunctionalInterface +// public interface CacheValidator +// { +// boolean isCacheValid(DhSectionPos sectionPos, long timestamp); +// } +// @FunctionalInterface +// public interface CacheSourceProducer +// { +// CompletableFuture getSourceFuture(DhSectionPos sectionPos); +// } +// CacheValidator validator; +// CacheSourceProducer source; + private final RenderFileHandler fileHandler; + private boolean doesFileExist; + + + + /** Creates a new metaFile */ + public RenderMetaDataFile(RenderFileHandler fileHandler, DhSectionPos pos) throws IOException + { + super(fileHandler.computeRenderFilePath(pos), pos); + this.fileHandler = fileHandler; + LodUtil.assertTrue(this.metaData == null); + this.doesFileExist = false; + } + + /** Uses the existing metaFile */ + public RenderMetaDataFile(RenderFileHandler fileHandler, File path) throws IOException + { + super(path); + this.fileHandler = fileHandler; + LodUtil.assertTrue(this.metaData != null); + this.loader = AbstractRenderSourceLoader.getLoader(this.metaData.dataTypeId, this.metaData.loaderVersion); + if (this.loader == null) + { + throw new IOException("Invalid file: Data type loader not found: " + + this.metaData.dataTypeId + "(v" + this.metaData.loaderVersion + ")"); + } + this.dataType = this.loader.clazz; + this.doesFileExist = true; + } + + + + // FIXME: This can cause concurrent modification of LodRenderSource. + // Not sure if it will cause issues or not. + public void updateChunkIfNeeded(ChunkSizedData chunkData, IDhClientLevel level) + { + DhLodPos chunkPos = new DhLodPos((byte) (chunkData.dataDetail + 4), chunkData.x, chunkData.z); + LodUtil.assertTrue(this.pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos {} doesn't overlap with section {}", chunkPos, pos); + + CompletableFuture source = this._readCached(this.data.get()); + if (source == null) + { + return; + } + + source.thenAccept((renderSource) -> renderSource.fastWrite(chunkData, level)); + } + + public CompletableFuture flushAndSave(ExecutorService renderCacheThread) + { + if (!path.exists()) + { + return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist. + } + + CompletableFuture source = this._readCached(this.data.get()); + if (source == null) + { + return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save. + } + + return source.thenAccept((a) -> { }); // Otherwise, wait for the data to be read (which also flushes changes to the file). + } + + // Suppress casting of CompletableFuture to CompletableFuture + @SuppressWarnings("unchecked") + private CompletableFuture _readCached(Object obj) + { + // Has file cached in RAM and not freed yet. + if ((obj instanceof SoftReference)) + { + Object inner = ((SoftReference) obj).get(); + if (inner != null) + { + LodUtil.assertTrue(inner instanceof ILodRenderSource); + fileHandler.onReadRenderSourceFromCache(this, (ILodRenderSource) inner); + return CompletableFuture.completedFuture((ILodRenderSource) inner); + } + } + + //==== Cached file out of scope. ==== + // Someone is already trying to complete it. so just return the obj. + if ((obj instanceof CompletableFuture)) + { + return (CompletableFuture) obj; + } + return null; + } + + // Cause: Generic Type runtime casting cannot safety check it. + // However, the Union type ensures the 'data' should only contain the listed type. + public CompletableFuture loadOrGetCached(Executor fileReaderThreads, IDhLevel level) + { + Object obj = this.data.get(); + + CompletableFuture cached = this._readCached(obj); + if (cached != null) + { + return cached; + } + + // Create an empty and non-completed future. + // Note: I do this before actually filling in the future so that I can ensure only + // one task is submitted to the thread pool. + CompletableFuture future = new CompletableFuture<>(); + + // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( + boolean worked = this.data.compareAndSet(obj, future); + if (!worked) + { + return this.loadOrGetCached(fileReaderThreads, level); + } + + // Now, there should only ever be one thread at a time here due to the CAS operation above. + + + // After cas. We are in exclusive control. + if (!this.doesFileExist) + { + this.fileHandler.onCreateRenderFile(this) + .thenApply((data) -> + { + this.metaData = makeMetaData(data); + return data; + }) + .thenApply((d) -> this.fileHandler.onRenderFileLoaded(d, this)) + .whenComplete((v, e) -> + { + if (e != null) + { + LOGGER.error("Uncaught error on creation {}: ", this.path, e); + future.complete(null); + this.data.set(null); + } + else + { + future.complete(v); + //new DataObjTracker(v); //TODO: Obj Tracker??? For debug? + this.data.set(new SoftReference<>(v)); + } + }); + } + else + { + CompletableFuture.supplyAsync(() -> + { + if (this.metaData == null) + { + throw new IllegalStateException("Meta data not loaded!"); + } + + // Load the file. + ILodRenderSource data; + data = this.fileHandler.onLoadingRenderFile(this); + if (data == null) + { + try (FileInputStream fio = getDataContent()) + { + data = this.loader.loadRender(this, fio, level); + } + catch (IOException e) + { + throw new CompletionException(e); + } + } + data = this.fileHandler.onRenderFileLoaded(data, this); + return data; + }, fileReaderThreads) + .whenComplete((f, e) -> + { + if (e != null) + { + LOGGER.error("Error loading file {}: ", this.path, e); + future.complete(null); + this.data.set(null); + } + else + { + future.complete(f); + this.data.set(new SoftReference<>(f)); + } + }); + } + return future; + } + + private static MetaData makeMetaData(ILodRenderSource data) + { + AbstractRenderSourceLoader loader = AbstractRenderSourceLoader.getLoader(data.getClass(), data.getRenderVersion()); + return new MetaData(data.getSectionPos(), -1, -1, + data.getDataDetail(), loader == null ? 0 : loader.renderTypeId, data.getRenderVersion()); + } + + private FileInputStream getDataContent() throws IOException + { + FileInputStream fin = new FileInputStream(this.path); + int toSkip = METADATA_SIZE; + while (toSkip > 0) + { + long skipped = fin.skip(toSkip); + if (skipped == 0) + { + throw new IOException("Invalid file: Failed to skip metadata."); + } + toSkip -= skipped; + } + if (toSkip != 0) + { + throw new IOException("File IO Error: Failed to skip metadata."); + } + return fin; + } + + public void save(ILodRenderSource data, IDhClientLevel level) + { + if (data.isEmpty()) + { + if (this.path.exists()) + { + if (!this.path.delete()) + { + LOGGER.warn("Failed to delete render file at {}", this.path); + } + } + this.doesFileExist = false; + } + else + { + LOGGER.info("Saving updated render file v[{}] at sect {}", this.metaData.dataVersion.get(), this.pos); + try + { + super.writeData((out) -> data.saveRender(level, this, out)); + this.doesFileExist = true; + } + catch (IOException e) + { + LOGGER.error("Failed to save updated render file at {} for sect {}", this.path, this.pos, e); + } + } + } + +} diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaFile.java deleted file mode 100644 index de06ea512..000000000 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaFile.java +++ /dev/null @@ -1,218 +0,0 @@ -package com.seibel.lod.core.file.renderfile; - -import com.seibel.lod.core.datatype.ILodDataSource; -import com.seibel.lod.core.datatype.ILodRenderSource; -import com.seibel.lod.core.datatype.AbstractRenderSourceLoader; -import com.seibel.lod.core.datatype.full.ChunkSizedData; -import com.seibel.lod.core.level.IDhClientLevel; -import com.seibel.lod.core.level.IDhLevel; -import com.seibel.lod.core.pos.DhLodPos; -import com.seibel.lod.core.file.subDimMatching.MetaFile; -import com.seibel.lod.core.pos.DhSectionPos; -import com.seibel.lod.core.logging.DhLoggerBuilder; -import com.seibel.lod.core.util.LodUtil; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.lang.ref.SoftReference; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicReference; - -public class RenderMetaFile extends MetaFile -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(RenderMetaFile.class.getSimpleName()); - - //private final IClientLevel level; - - public AbstractRenderSourceLoader loader; - public Class dataType; - - // The '?' type should either be: - // SoftReference, or - File that may still be loaded - // CompletableFuture,or - File that is being loaded - // null - Nothing is loaded or being loaded - AtomicReference data = new AtomicReference<>(null); - - //FIXME: This can cause concurrent modification of LodRenderSource. - // Not sure if it will cause issues or not. - public void updateChunkIfNeeded(ChunkSizedData chunkData, IDhClientLevel level) { - DhLodPos chunkPos = new DhLodPos((byte) (chunkData.dataDetail + 4), chunkData.x, chunkData.z); - LodUtil.assertTrue(pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos {} doesn't overlap with section {}", chunkPos, pos); - - CompletableFuture source = _readCached(data.get()); - if (source == null) return; - source.thenAccept((renderSource) -> renderSource.fastWrite(chunkData, level)); - } - - public CompletableFuture flushAndSave(ExecutorService renderCacheThread) { - if (!path.exists()) return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist. - CompletableFuture source = _readCached(data.get()); - if (source == null) return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save. - return source.thenAccept((a)->{}); // Otherwise, wait for the data to be read (which also flushes changes to the file). - } - - @FunctionalInterface - public interface CacheValidator { - boolean isCacheValid(DhSectionPos sectionPos, long timestamp); - } - @FunctionalInterface - public interface CacheSourceProducer { - CompletableFuture getSourceFuture(DhSectionPos sectionPos); - } - CacheValidator validator; - CacheSourceProducer source; - final RenderFileHandler handler; - private boolean doesFileExist; - - - // Create a new metaFile - public RenderMetaFile(RenderFileHandler handler, DhSectionPos pos) throws IOException { - super(handler.computeRenderFilePath(pos), pos); - this.handler = handler; - LodUtil.assertTrue(metaData == null); - doesFileExist = false; - } - - public RenderMetaFile(RenderFileHandler handler, File path) throws IOException { - super(path); - this.handler = handler; - LodUtil.assertTrue(metaData != null); - loader = AbstractRenderSourceLoader.getLoader(metaData.dataTypeId, metaData.loaderVersion); - if (loader == null) { - throw new IOException("Invalid file: Data type loader not found: " - + metaData.dataTypeId + "(v" + metaData.loaderVersion + ")"); - } - dataType = loader.clazz; - doesFileExist = true; - } - - // Suppress casting of CompletableFuture to CompletableFuture - @SuppressWarnings("unchecked") - private CompletableFuture _readCached(Object obj) { - // Has file cached in RAM and not freed yet. - if ((obj instanceof SoftReference)) { - Object inner = ((SoftReference)obj).get(); - if (inner != null) { - LodUtil.assertTrue(inner instanceof ILodRenderSource); - handler.onReadRenderSourceFromCache(this, (ILodRenderSource) inner); - return CompletableFuture.completedFuture((ILodRenderSource)inner); - } - } - - //==== Cached file out of scope. ==== - // Someone is already trying to complete it. so just return the obj. - if ((obj instanceof CompletableFuture)) { - return (CompletableFuture)obj; - } - return null; - } - - // Cause: Generic Type runtime casting cannot safety check it. - // However, the Union type ensures the 'data' should only contain the listed type. - public CompletableFuture loadOrGetCached(Executor fileReaderThreads, IDhLevel level) { - Object obj = data.get(); - - CompletableFuture cached = _readCached(obj); - if (cached != null) return cached; - - // Create an empty and non-completed future. - // Note: I do this before actually filling in the future so that I can ensure only - // one task is submitted to the thread pool. - CompletableFuture future = new CompletableFuture<>(); - - // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( - boolean worked = data.compareAndSet(obj, future); - if (!worked) return loadOrGetCached(fileReaderThreads, level); - - // Now, there should only ever be one thread at a time here due to the CAS operation above. - - - // After cas. We are in exclusive control. - if (!doesFileExist) { - handler.onCreateRenderFile(this) - .thenApply((data) -> { - metaData = makeMetaData(data); - return data; - }) - .thenApply((d) -> handler.onRenderFileLoaded(d, this)) - .whenComplete((v, e) -> { - if (e != null) { - LOGGER.error("Uncaught error on creation {}: ", path, e); - future.complete(null); - data.set(null); - } else { - future.complete(v); - //new DataObjTracker(v); //TODO: Obj Tracker??? For debug? - data.set(new SoftReference<>(v)); - } - }); - } else { - CompletableFuture.supplyAsync(() -> { - if (metaData == null) - throw new IllegalStateException("Meta data not loaded!"); - // Load the file. - ILodRenderSource data; - data = handler.onLoadingRenderFile(this); - if (data == null) { - try (FileInputStream fio = getDataContent()) { - data = loader.loadRender(this, fio, level); - } catch (IOException e) { - throw new CompletionException(e); - } - } - data = handler.onRenderFileLoaded(data, this); - return data; - }, fileReaderThreads) - .whenComplete((f, e) -> { - if (e != null) { - LOGGER.error("Error loading file {}: ", path, e); - future.complete(null); - data.set(null); - } else { - future.complete(f); - data.set(new SoftReference<>(f)); - } - }); - } - return future; - } - - private static MetaData makeMetaData(ILodRenderSource data) { - AbstractRenderSourceLoader loader = AbstractRenderSourceLoader.getLoader(data.getClass(), data.getRenderVersion()); - return new MetaData(data.getSectionPos(), -1, -1, - data.getDataDetail(), loader == null ? 0 : loader.renderTypeId, data.getRenderVersion()); - } - - private FileInputStream getDataContent() throws IOException { - FileInputStream fin = new FileInputStream(path); - int toSkip = METADATA_SIZE; - while (toSkip > 0) { - long skipped = fin.skip(toSkip); - if (skipped == 0) { - throw new IOException("Invalid file: Failed to skip metadata."); - } - toSkip -= skipped; - } - if (toSkip != 0) { - throw new IOException("File IO Error: Failed to skip metadata."); - } - return fin; - } - - public void save(ILodRenderSource data, IDhClientLevel level) { - if (data.isEmpty()) { - if (path.exists()) if (!path.delete()) LOGGER.warn("Failed to delete render file at {}", path); - doesFileExist = false; - } else { - LOGGER.info("Saving updated render file v[{}] at sect {}", metaData.dataVersion.get(), pos); - try { - super.writeData((out) -> data.saveRender(level, this, out)); - doesFileExist = true; - } catch (IOException e) { - LOGGER.error("Failed to save updated render file at {} for sect {}", path, pos, e); - } - } - } -} diff --git a/core/src/main/java/com/seibel/lod/core/file/subDimMatching/MetaFile.java b/core/src/main/java/com/seibel/lod/core/file/subDimMatching/MetaFile.java deleted file mode 100644 index 3f1c8e98d..000000000 --- a/core/src/main/java/com/seibel/lod/core/file/subDimMatching/MetaFile.java +++ /dev/null @@ -1,197 +0,0 @@ -package com.seibel.lod.core.file.subDimMatching; - -import java.io.*; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.util.concurrent.atomic.AtomicLong; -import java.util.zip.Adler32; -import java.util.zip.CheckedOutputStream; - -import com.seibel.lod.core.pos.DhSectionPos; -import com.seibel.lod.core.util.objects.UnclosableOutputStream; -import com.seibel.lod.core.logging.DhLoggerBuilder; -import com.seibel.lod.core.util.LodUtil; -import org.apache.logging.log4j.Logger; - -/** - * Used size: 40 bytes
- * Remaining space: 24 bytes
- * Total size: 64 bytes


- * - * - * Metadata format:

- * - * 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signals the metadata format)
- * 4 bytes: section X position
- * 4 bytes: section Y position (Unused, for future proofing)
- * 4 bytes: section Z position

- * - * 4 bytes: data checksum
//TODO: Implement checksum - * 1 byte: section detail level
- * 1 byte: data detail level // Note: not sure if this is needed
- * 1 byte: loader version
- * 1 byte: unused

- * - * 8 bytes: datatype identifier

- * - * 8 bytes: data version - */ -public class MetaFile -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - - - - public static final int METADATA_SIZE = 64; - public static final int METADATA_RESERVED_SIZE = 24; - public static final int METADATA_MAGIC_BYTES = 0x44_48_76_30; - - /** Currently set to false because for some reason Window is throwing PermissionDeniedException when trying to atomic replace a file... */ - public static final boolean USE_ATOMIC_MOVE_REPLACE = false; - - public final DhSectionPos pos; - - public File path; - - @FunctionalInterface - public interface IOConsumer { - void accept(T t) throws IOException; - } - - public static class MetaData { - public DhSectionPos pos; - public int checksum; - public AtomicLong dataVersion; - public byte dataLevel; - //Loader stuff - public long dataTypeId; - public byte loaderVersion; - - public MetaData(DhSectionPos pos, int checksum, long dataVersion, byte dataLevel, long dataTypeId, byte loaderVersion) { - this.pos = pos; - this.checksum = checksum; - this.dataVersion = new AtomicLong(dataVersion); - this.dataLevel = dataLevel; - this.dataTypeId = dataTypeId; - this.loaderVersion = loaderVersion; - } - } - public volatile MetaData metaData = null; - private static MetaData readMeta(File file) throws IOException { - try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { - ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE); - channel.read(buffer, 0); - channel.close(); - buffer.flip(); - - int magic = buffer.getInt(); - if (magic != METADATA_MAGIC_BYTES) { - throw new IOException("Invalid file: Magic bytes check failed."); - } - int x = buffer.getInt(); - int y = buffer.getInt(); // Unused - int z = buffer.getInt(); - int checksum = buffer.getInt(); - byte detailLevel = buffer.get(); - byte dataLevel = buffer.get(); - byte loaderVersion = buffer.get(); - byte unused = buffer.get(); - long dataTypeId = buffer.getLong(); - long timestamp = buffer.getLong(); - LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE); - DhSectionPos dataPos = new DhSectionPos(detailLevel, x, z); - return new MetaData(dataPos, checksum, timestamp, dataLevel, dataTypeId, loaderVersion); - } - } - - private void validateFile() throws IOException { - if (!path.exists()) throw new IOException("File missing"); - if (!path.isFile()) throw new IOException("Not a file"); - if (!path.canRead()) throw new IOException("File not readable"); - if (!path.canWrite()) throw new IOException("File not writable"); - } - - // Create a metaFile in this path. If the path has a file, throws FileAlreadyExistsException - protected MetaFile(File path, DhSectionPos pos) throws IOException { - this.path = path; - this.pos = pos; - if (path.exists()) throw new FileAlreadyExistsException(path.toString()); - } - // Load a metaFile in this path - protected MetaFile(File path) throws IOException { - this.path = path; - if (!path.exists()) throw new FileNotFoundException("File not found at " + path); - validateFile(); - metaData = readMeta(path); - pos = metaData.pos; - } - - protected void loadMetaData() throws IOException { - validateFile(); - metaData = readMeta(path); - if (!metaData.pos.equals(pos)) { - LOGGER.warn("The file is from a different location than expected! Expects {} but got {}. Ignoring file tag.", pos, metaData.pos); - metaData.pos = pos; - } - } - - protected void writeData(IOConsumer dataWriter) throws IOException { - LodUtil.assertTrue(metaData != null); - if (path.exists()) validateFile(); - File writerFile; - if (USE_ATOMIC_MOVE_REPLACE) { - writerFile = new File(path.getPath() + ".tmp"); - writerFile.deleteOnExit(); - } else { - writerFile = path; - } - - try (FileChannel file = FileChannel.open(writerFile.toPath(), - StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { - { - file.position(METADATA_SIZE); - int checksum; - try (OutputStream channelOut = new UnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel - BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer? - CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) { // TODO: Is Adler32 ok? - dataWriter.accept(checkedOut); - checksum = (int) checkedOut.getChecksum().getValue(); - } - file.position(0); - // Write metadata - ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE); - buff.putInt(METADATA_MAGIC_BYTES); - buff.putInt(pos.sectionX); - buff.putInt(Integer.MIN_VALUE); // Unused - buff.putInt(pos.sectionZ); - buff.putInt(checksum); - buff.put(pos.sectionDetail); - buff.put(metaData.dataLevel); - buff.put(metaData.loaderVersion); - buff.put(Byte.MIN_VALUE); // Unused - buff.putLong(metaData.dataTypeId); - buff.putLong(metaData.dataVersion.get()); - LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE); - buff.flip(); - file.write(buff); - } - file.close(); - if (USE_ATOMIC_MOVE_REPLACE) { - // Atomic move / replace the actual file - Files.move(writerFile.toPath(), path.toPath(), StandardCopyOption.ATOMIC_MOVE); - } - } finally { - try { - if (USE_ATOMIC_MOVE_REPLACE && writerFile.exists()) { - boolean i = writerFile.delete(); // Delete temp file. Ignore errors if fails. - } - } catch (Exception ignored) {} - } - } -}