diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index cbd4a4c80..f87e17503 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -19,7 +19,6 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.google.common.collect.HashMultimap; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; @@ -33,12 +32,11 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.util.FileScanUtil; +import com.seibel.distanthorizons.core.util.MetaFileScanUtil; import com.seibel.distanthorizons.core.util.FileUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; @@ -48,8 +46,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; -import static com.seibel.distanthorizons.core.util.FileScanUtil.LOD_FILE_POSTFIX; - public class FullDataFileHandler implements IFullDataSourceProvider { public static final boolean USE_LAZY_LOADING = true; @@ -59,18 +55,17 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected static ExecutorService fileHandlerThreadPool; protected static ConfigChangeListener configListener; - private final ConcurrentHashMap unloadedFiles = new ConcurrentHashMap<>(); - private final ConcurrentHashMap fileBySectionPos = new ConcurrentHashMap<>(); - public void ForEachFile(Consumer consumer) { this.fileBySectionPos.values().forEach(consumer); } - - private LinkedList> onUpdatedListeners = new LinkedList<>(); + private final ConcurrentHashMap unloadedFileBySectionPos = new ConcurrentHashMap<>(); + /** contains the loaded {@link FullDataMetaFile}'s */ + private final ConcurrentHashMap metaFileBySectionPos = new ConcurrentHashMap<>(); protected final IDhLevel level; protected final File saveDir; - protected final AtomicInteger topDetailLevel = new AtomicInteger(0); + protected final AtomicInteger topDetailLevelRef = new AtomicInteger(0); protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; + //=============// // constructor // //=============// @@ -83,133 +78,85 @@ public class FullDataFileHandler implements IFullDataSourceProvider { LOGGER.warn("Unable to create full data folder, file saving may fail."); } - FileScanUtil.scanFullDataFiles(saveStructure, level.getLevelWrapper(), this); + MetaFileScanUtil.scanFullDataFiles(saveStructure, level.getLevelWrapper(), this); } - // constructor helpers // + @Override + public void addScannedFiles(Collection detectedFiles) + { + MetaFileScanUtil.CreateMetadataFunc createMetadataFunc = (file) -> new FullDataMetaFile(this, this.level, file); + + MetaFileScanUtil.AddUnloadedFileFunc addUnloadedFileFunc = (pos, file) -> + { + this.unloadedFileBySectionPos.put(pos, file); + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + }; + MetaFileScanUtil.AddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) -> + { + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + this.metaFileBySectionPos.put(pos, (FullDataMetaFile) loadedMetaFile); + }; + + + MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, FullDataMetaFile.FILE_SUFFIX, + createMetadataFunc, + addUnloadedFileFunc, addLoadedMetaFileFunc); + } + + + + //===============// + // file handling // + //===============// /** - * Caller must ensure that this method is called only once, - * and that the {@link FullDataFileHandler} is not used before this method is called. + * Returns the {@link IFullDataSource} for the given section position.
+ * The returned data source may be null.

+ * + * For now, if result is null, it prob means error has occurred when loading or creating the file object.

+ * + * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ @Override - public void addScannedFile(Collection detectedFiles) + public CompletableFuture readAsync(DhSectionPos pos) { - if (USE_LAZY_LOADING) + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true); + if (metaFile == null) { - lazyAddScannedFile(detectedFiles); - } - else - { - immediateAddScannedFile(detectedFiles); - } - } - - private void lazyAddScannedFile(Collection detectedFiles) - { - for (File file : detectedFiles) - { - try - { - DhSectionPos pos = decodePositionByFile(file); - if (pos != null) - { - unloadedFiles.put(pos, file); - this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); - } - } - catch (Exception e) - { - LOGGER.error("Failed to read data meta file at " + file + ": ", e); - FileUtil.renameCorruptedFile(file); - } - } - } - - private void immediateAddScannedFile(Collection detectedFiles) - { - HashMultimap filesByPos = HashMultimap.create(); - { // Sort files by pos. - for (File file : detectedFiles) - { - try - { - FullDataMetaFile metaFile = new FullDataMetaFile(this, this.level, file); - filesByPos.put(metaFile.pos, metaFile); - } - catch (IOException e) - { - LOGGER.error("Failed to read data meta file at " + file + ": ", e); - FileUtil.renameCorruptedFile(file); - } - } + return CompletableFuture.completedFuture(null); } - // Warn for multiple files with the same pos, and then select the one with the latest timestamp. - for (DhSectionPos pos : filesByPos.keySet()) - { - Collection metaFiles = filesByPos.get(pos); - FullDataMetaFile fileToUse; - if (metaFiles.size() > 1) - { -// fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get())); - - fileToUse = Collections.max(metaFiles, Comparator.comparingLong(fullDataMetaFile -> fullDataMetaFile.file.lastModified())); + + // future wrapper necessary in order to handle file read errors + CompletableFuture futureWrapper = new CompletableFuture<>(); + metaFile.loadOrGetCachedDataSourceAsync().exceptionally((e) -> { - StringBuilder sb = new StringBuilder(); - sb.append("Multiple files with the same pos: "); - sb.append(pos); - sb.append("\n"); - for (FullDataMetaFile metaFile : metaFiles) - { - sb.append("\t"); - sb.append(metaFile.file); - sb.append("\n"); - } - sb.append("\tUsing: "); - sb.append(fileToUse.file); - sb.append("\n"); - sb.append("(Other files will be renamed by appending \".old\" to their name.)"); - LOGGER.warn(sb.toString()); + FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, metaFile, e); - // Rename all other files with the same pos to .old - for (FullDataMetaFile metaFile : metaFiles) - { - if (metaFile == fileToUse) - { - continue; - } - File oldFile = new File(metaFile.file + ".old"); - try - { - if (!metaFile.file.renameTo(oldFile)) - { - throw new RuntimeException("Renaming failed"); - } - } - catch (Exception e) - { - LOGGER.error("Failed to rename file: " + metaFile.file + " to " + oldFile, e); - } - } - } - } - else - { - fileToUse = metaFiles.iterator().next(); - } - // Add file to the list of files. - this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, fileToUse.pos.sectionDetailLevel)); - this.fileBySectionPos.put(pos, fileToUse); - } + futureWrapper.completeExceptionally(e); + return null; // return value doesn't matter + }) + .whenComplete((dataSource, e) -> + { + futureWrapper.complete(dataSource); + }); + + return futureWrapper; } + @Override + public FullDataMetaFile getFileIfExist(DhSectionPos pos) { return this.getLoadOrMakeFile(pos, false); } protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile) { - FullDataMetaFile metaFile = this.fileBySectionPos.get(pos); - if (metaFile != null) return metaFile; + FullDataMetaFile metaFile = this.metaFileBySectionPos.get(pos); + if (metaFile != null) + { + return metaFile; + } - File fileToLoad = this.unloadedFiles.get(pos); + + File fileToLoad = this.unloadedFileBySectionPos.get(pos); // File does exist, but not loaded yet. if (fileToLoad != null) { @@ -218,7 +165,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider // Double check locking for loading file, as loading file means also loading the metadata, which // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. - metaFile = this.fileBySectionPos.get(pos); + metaFile = this.metaFileBySectionPos.get(pos); if (metaFile != null) { return metaFile; // someone else loaded it already. @@ -227,8 +174,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider try { metaFile = new FullDataMetaFile(this, this.level, fileToLoad); - this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); - this.fileBySectionPos.put(pos, metaFile); + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + this.metaFileBySectionPos.put(pos, metaFile); return metaFile; } catch (IOException e) @@ -238,7 +185,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider } finally { - this.unloadedFiles.remove(pos); + this.unloadedFileBySectionPos.remove(pos); } } } @@ -262,10 +209,10 @@ public class FullDataFileHandler implements IFullDataSourceProvider return null; } - this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); // This is a CAS with expected null value. - FullDataMetaFile metaFileCas = this.fileBySectionPos.putIfAbsent(pos, metaFile); + FullDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile); return metaFileCas == null ? metaFile : metaFileCas; } @@ -304,7 +251,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider } // check if a file for this pos exists, either loaded and unloaded - if (this.fileBySectionPos.containsKey(subPos) || this.unloadedFiles.containsKey(subPos)) + if (this.metaFileBySectionPos.containsKey(subPos) || this.unloadedFileBySectionPos.containsKey(subPos)) { allEmpty = false; break outerLoop; @@ -334,13 +281,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos)) { // load the file if it isn't already - if (this.unloadedFiles.containsKey(childPos)) + if (this.unloadedFileBySectionPos.containsKey(childPos)) { this.getLoadOrMakeFile(childPos, true); } - FullDataMetaFile metaFile = this.fileBySectionPos.get(childPos); + FullDataMetaFile metaFile = this.metaFileBySectionPos.get(childPos); if (metaFile != null) { // we have reached a populated leaf node in the quad tree @@ -359,42 +306,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider } } + public void ForEachFile(Consumer consumer) { this.metaFileBySectionPos.values().forEach(consumer); } - /** - * Returns the {@link IFullDataSource} for the given section position.
- * The returned data source may be null.

- * - * For now, if result is null, it prob means error has occurred when loading or creating the file object.

- * - * This call is concurrent. I.e. it supports being called by multiple threads at the same time. - */ - @Override - public CompletableFuture readAsync(DhSectionPos pos) - { - this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); - FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true); - if (metaFile == null) - { - return CompletableFuture.completedFuture(null); - } - - - // future wrapper necessary in order to handle file read errors - CompletableFuture futureWrapper = new CompletableFuture<>(); - metaFile.loadOrGetCachedDataSourceAsync().exceptionally((e) -> - { - FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, metaFile, e); - - futureWrapper.completeExceptionally(e); - return null; // return value doesn't matter - }) - .whenComplete((dataSource, e) -> - { - futureWrapper.complete(dataSource); - }); - - return futureWrapper; - } + + + //=============// + // data saving // + //=============// /** This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ @Override @@ -408,14 +326,14 @@ public class FullDataFileHandler implements IFullDataSourceProvider } private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData) { - FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos); + FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos); if (metaFile != null) { // there is a file for this position metaFile.addToWriteQueue(chunkData); } - if (sectionPos.sectionDetailLevel <= this.topDetailLevel.get()) + if (sectionPos.sectionDetailLevel <= this.topDetailLevelRef.get()) { // recursively attempt to get the meta file for this position this.writeChunkDataToMetaFile(sectionPos.getParentPos(), chunkData); @@ -427,7 +345,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider public CompletableFuture flushAndSave() { ArrayList> futures = new ArrayList<>(); - for (FullDataMetaFile metaFile : this.fileBySectionPos.values()) + for (FullDataMetaFile metaFile : this.metaFileBySectionPos.values()) { futures.add(metaFile.flushAndSaveAsync()); } @@ -437,7 +355,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider @Override public CompletableFuture flushAndSave(DhSectionPos sectionPos) { - FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos); + FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos); if (metaFile == null) { return CompletableFuture.completedFuture(null); @@ -446,11 +364,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider } - @Override - public synchronized void addOnUpdatedListener(Consumer listener) - { - this.onUpdatedListeners.add(listener); - } + protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { @@ -526,7 +440,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider FileUtil.renameCorruptedFile(metaFile.file); // remove the FullDataMetaFile since the old one was corrupted - this.fileBySectionPos.remove(pos); + this.metaFileBySectionPos.remove(pos); // create a new FullDataMetaFile to write new data to return this.getLoadOrMakeFile(pos, true); } @@ -537,10 +451,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider Consumer onUpdated, Function updater) { boolean changed = updater.apply(source); -// if (changed) -// { -// metaData.dataVersion.incrementAndGet(); -// } if (source instanceof IIncompleteFullDataSource) { @@ -556,17 +466,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider return CompletableFuture.completedFuture(source); } - @Override - public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + LOD_FILE_POSTFIX); } - - @Nullable - public DhSectionPos decodePositionByFile(File file) - { - String fileName = file.getName(); - if (!fileName.endsWith(LOD_FILE_POSTFIX)) return null; - fileName = fileName.substring(0, fileName.length() - 4); - return DhSectionPos.deserialize(fileName); - } //==========================// @@ -619,12 +518,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider @Override public ExecutorService getIOExecutor() { return fileHandlerThreadPool; } - @Override - public FullDataMetaFile getFileIfExist(DhSectionPos pos) - { - return getLoadOrMakeFile(pos, false); - } - //=========// @@ -632,9 +525,15 @@ public class FullDataFileHandler implements IFullDataSourceProvider //=========// @Override - public void close() - { - FullDataMetaFile.debugPhantomLifeCycleCheck(); - } + public void close() { FullDataMetaFile.debugPhantomLifeCycleCheck(); } + + + + //================// + // helper methods // + //================// + + @Override + public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + FullDataMetaFile.FILE_SUFFIX); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java index 2158b28e5..6fe90ab5f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java @@ -51,8 +51,12 @@ import org.apache.logging.log4j.Logger; */ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable { + public static final String FILE_SUFFIX = ".lod"; + private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName()); + + private final IDhLevel level; private final IFullDataSourceProvider fullDataSourceProvider; public boolean doesFileExist; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index b75608115..76056359c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -57,18 +57,6 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler - //======// - // data // - //======// - - @Override - public CompletableFuture readAsync(DhSectionPos pos) - { - return super.readAsync(pos); - } - - - //==================// // generation queue // //==================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index a295557dd..8055e34a6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -33,15 +33,13 @@ import java.util.function.Function; public interface IFullDataSourceProvider extends AutoCloseable { - void addScannedFile(Collection detectedFiles); + void addScannedFiles(Collection detectedFiles); CompletableFuture readAsync(DhSectionPos pos); void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData); CompletableFuture flushAndSave(); CompletableFuture flushAndSave(DhSectionPos sectionPos); - void addOnUpdatedListener(Consumer listener); - //long getCacheVersion(DhSectionPos sectionPos); //boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java index bbb3dc2dc..db6e5fb3c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java @@ -35,14 +35,13 @@ import java.util.concurrent.CompletableFuture; */ public interface ILodRenderSourceProvider extends AutoCloseable { - CompletableFuture readAsync(DhSectionPos pos); void addScannedFiles(Collection detectedFiles); + + CompletableFuture readAsync(DhSectionPos pos); + void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData); CompletableFuture flushAndSaveAsync(); - /** Returns true if the data was refreshed, false otherwise */ - //boolean refreshRenderSource(ColumnRenderSource source); - /** Deletes any data stored in the render cache so it can be re-created */ void deleteRenderCache(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java index f88b82850..b829c4c3f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java @@ -56,6 +56,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + public static final String FILE_SUFFIX = ".rlod"; public static final boolean ALWAYS_INVALIDATE_CACHE = false; public static final long RENDER_SOURCE_TYPE_ID = ColumnRenderSource.TYPE_ID; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index f421e45cf..57d3e8987 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -19,7 +19,6 @@ package com.seibel.distanthorizons.core.file.renderfile; -import com.google.common.collect.HashMultimap; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -29,12 +28,11 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.util.FileScanUtil; +import com.seibel.distanthorizons.core.util.MetaFileScanUtil; import com.seibel.distanthorizons.core.util.FileUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; @@ -42,8 +40,6 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX; - public class RenderSourceFileHandler implements ILodRenderSourceProvider { public static final boolean USE_LAZY_LOADING = true; @@ -60,7 +56,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider private final IDhClientLevel clientLevel; private final File saveDir; /** This is the lowest (highest numeric) detail level that this {@link RenderSourceFileHandler} is keeping track of. */ - AtomicInteger topDetailLevel = new AtomicInteger(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + AtomicInteger topDetailLevelRef = new AtomicInteger(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); private final IFullDataSourceProvider fullDataSourceProvider; private final WeakHashMap, ETaskType> taskTracker = new WeakHashMap<>(); @@ -85,136 +81,35 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log); - FileScanUtil.scanRenderFiles(saveStructure, clientLevel.getLevelWrapper(), this); + MetaFileScanUtil.scanRenderFiles(saveStructure, clientLevel.getLevelWrapper(), this); } /** * Caller must ensure that this method is called only once, * and that the given files are not used before this method is called.

* - * Used by {@link FileScanUtil#scanRenderFiles(AbstractSaveStructure, ILevelWrapper, ILodRenderSourceProvider)} + * Used by {@link MetaFileScanUtil#scanRenderFiles(AbstractSaveStructure, ILevelWrapper, ILodRenderSourceProvider)} */ @Override public void addScannedFiles(Collection detectedFiles) { - if (USE_LAZY_LOADING) - { - this.lazyAddScannedFile(detectedFiles); - } - else - { - this.immediateAddScannedFile(detectedFiles); - } - } - private void lazyAddScannedFile(Collection detectedFiles) - { - for (File file : detectedFiles) - { - if (file == null || !file.exists()) - { - // can rarely happen if the user rapidly travels between dimensions - LOGGER.warn("Null or non-existent render file: " + ((file != null) ? file.getPath() : "NULL")); - continue; - } - - - try - { - DhSectionPos pos = this.decodePositionFromFileName(file); - if (pos != null) - { - this.unloadedFileBySectionPos.put(pos, file); - this.topDetailLevel.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.sectionDetailLevel)); - } - } - catch (Exception e) - { - LOGGER.error("Failed to read data meta file at " + file + ": ", e); - FileUtil.renameCorruptedFile(file); - } - } - } - private void immediateAddScannedFile(Collection newRenderFiles) - { - HashMultimap filesByPos = HashMultimap.create(); + MetaFileScanUtil.CreateMetadataFunc createMetadataFunc = (file) -> RenderMetaDataFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, file); - // Sort files by pos. - for (File file : newRenderFiles) + MetaFileScanUtil.AddUnloadedFileFunc addUnloadedFileFunc = (pos, file) -> { - try - { - RenderMetaDataFile metaFile = RenderMetaDataFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, file); - filesByPos.put(metaFile.pos, metaFile); - } - catch (IOException e) - { - LOGGER.error("Failed to read render meta file at [" + file + "]. Error: ", e); - FileUtil.renameCorruptedFile(file); - } - } + this.unloadedFileBySectionPos.put(pos, file); + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + }; + MetaFileScanUtil.AddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) -> + { + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel)); + this.metaFileBySectionPos.put(pos, (RenderMetaDataFile) loadedMetaFile); + }; - // Warn for multiple files with the same pos, and then select the one with the latest timestamp. - for (DhSectionPos pos : filesByPos.keySet()) - { - Collection metaFiles = filesByPos.get(pos); - RenderMetaDataFile fileToUse; - if (metaFiles.size() > 1) - { - //fileToUse = metaFiles.stream().findFirst().orElse(null); // use the first file in the list - - // use the file's last modified date - fileToUse = Collections.max(metaFiles, Comparator.comparingLong(renderMetaDataFile -> - renderMetaDataFile.file.lastModified())); - -// fileToUse = Collections.max(metaFiles, Comparator.comparingLong(renderMetaDataFile -> -// renderMetaDataFile.metaData.dataVersion.get())); - { - StringBuilder sb = new StringBuilder(); - sb.append("Multiple files with the same pos: "); - sb.append(pos); - sb.append("\n"); - for (RenderMetaDataFile metaFile : metaFiles) - { - sb.append("\t"); - sb.append(metaFile.file); - sb.append("\n"); - } - sb.append("\tUsing: "); - sb.append(fileToUse.file); - sb.append("\n"); - sb.append("(Other files will be renamed by appending \".old\" to their name.)"); - LOGGER.warn(sb.toString()); - - // Rename all other files with the same pos to .old - for (RenderMetaDataFile metaFile : metaFiles) - { - if (metaFile == fileToUse) - { - continue; - } - - File oldFile = new File(metaFile.file + ".old"); - try - { - if (!metaFile.file.renameTo(oldFile)) - throw new RuntimeException("Renaming failed"); - } - catch (Exception e) - { - LOGGER.error("Failed to rename file: [" + metaFile.file + "] to [" + oldFile + "]", e); - } - } - } - } - else - { - fileToUse = metaFiles.iterator().next(); - } - // Add this file to the list of files. - this.metaFileBySectionPos.put(pos, fileToUse); - // increase the lowest detail level if a new lower detail file is found - this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); - } + + MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, RenderMetaDataFile.FILE_SUFFIX, + createMetadataFunc, + addUnloadedFileFunc, addLoadedMetaFileFunc); } @@ -300,7 +195,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider try { metaFile = RenderMetaDataFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, fileToLoad); - this.topDetailLevel.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.sectionDetailLevel)); + this.topDetailLevelRef.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.sectionDetailLevel)); this.metaFileBySectionPos.put(pos, metaFile); return metaFile; } @@ -326,7 +221,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider // due to a rare issue where the file may already exist but isn't in the file list metaFile = RenderMetaDataFile.createFromExistingOrNewFile(this.clientLevel, this.fullDataSourceProvider, pos, this.computeRenderFilePath(pos)); - this.topDetailLevel.updateAndGet(newDetailLevel -> Math.max(newDetailLevel, pos.sectionDetailLevel)); + this.topDetailLevelRef.updateAndGet(newDetailLevel -> Math.max(newDetailLevel, pos.sectionDetailLevel)); // Compare And Swap to handle a concurrency issue where multiple threads created the same Meta File at the same time RenderMetaDataFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile); @@ -375,7 +270,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider } } - if (sectionDetailLevel < this.topDetailLevel.get()) + if (sectionDetailLevel < this.topDetailLevelRef.get()) { this.writeChunkDataToFileRecursively(chunk, (byte) (sectionDetailLevel + 1)); } @@ -483,16 +378,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider // helper methods // //================// - public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RENDER_FILE_POSTFIX); } - - @Nullable - public DhSectionPos decodePositionFromFileName(File file) - { - String fileName = file.getName(); - if (!fileName.endsWith(RENDER_FILE_POSTFIX)) return null; - fileName = fileName.substring(0, fileName.length() - RENDER_FILE_POSTFIX.length()); - return DhSectionPos.deserialize(fileName); - } + public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RenderMetaDataFile.FILE_SUFFIX); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java deleted file mode 100644 index 8db0eb414..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.util; - -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** Used to pull in the initial files used by both {@link IFullDataSourceProvider} and {@link ILodRenderSourceProvider}s. */ -public class FileScanUtil -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - public static final int MAX_SCAN_DEPTH = 5; - public static final String LOD_FILE_POSTFIX = ".lod"; - public static final String RENDER_FILE_POSTFIX = ".rlod"; - - - - public static void scanFullDataFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, IFullDataSourceProvider dataSourceProvider) - { - try (Stream pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) - { - List files = pathStream.filter( - path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile() - ).map(Path::toFile).collect(Collectors.toList()); - LOGGER.info("Found " + files.size() + " full data files for " + levelWrapper + " in " + saveStructure); - dataSourceProvider.addScannedFile(files); - } - catch (Exception e) - { - LOGGER.error("Failed to scan and collect full data files for " + levelWrapper + " in " + saveStructure, e); - } - } - - public static void scanRenderFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, ILodRenderSourceProvider renderSourceProvider) - { - try (Stream pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) - { - List files = pathStream.filter( - path -> path.toFile().getName().endsWith(RENDER_FILE_POSTFIX) && path.toFile().isFile() - ).map(Path::toFile).collect(Collectors.toList()); - LOGGER.info("Found " + files.size() + " render cache files for " + levelWrapper + " in " + saveStructure); - renderSourceProvider.addScannedFiles(files); - } - catch (Exception e) - { - LOGGER.error("Failed to scan and collect cache files for " + levelWrapper + " in " + saveStructure, e); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/MetaFileScanUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/MetaFileScanUtil.java new file mode 100644 index 000000000..18d002019 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/MetaFileScanUtil.java @@ -0,0 +1,257 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.util; + +import com.google.common.collect.HashMultimap; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; +import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; +import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider; +import com.seibel.distanthorizons.core.file.renderfile.RenderMetaDataFile; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Used to pull in the initial files used by both {@link IFullDataSourceProvider} and {@link ILodRenderSourceProvider}s. */ +public class MetaFileScanUtil +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + public static final int MAX_SCAN_DEPTH = 5; + + + + //===============// + // file scanning // + //===============// + + // file scanning means to find all File's in a given directory + + // TODO merge with the below method + public static void scanFullDataFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, IFullDataSourceProvider dataSourceProvider) + { + try (Stream pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) + { + List files = pathStream.filter( + path -> path.toFile().getName().endsWith(FullDataMetaFile.FILE_SUFFIX) && path.toFile().isFile() + ).map(Path::toFile).collect(Collectors.toList()); + LOGGER.info("Found " + files.size() + " full data files for " + levelWrapper + " in " + saveStructure); + dataSourceProvider.addScannedFiles(files); + } + catch (Exception e) + { + LOGGER.error("Failed to scan and collect full data files for " + levelWrapper + " in " + saveStructure, e); + } + } + + // TODO merge with the above method + public static void scanRenderFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, ILodRenderSourceProvider renderSourceProvider) + { + try (Stream pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) + { + List files = pathStream.filter( + path -> path.toFile().getName().endsWith(RenderMetaDataFile.FILE_SUFFIX) && path.toFile().isFile() + ).map(Path::toFile).collect(Collectors.toList()); + LOGGER.info("Found " + files.size() + " render cache files for " + levelWrapper + " in " + saveStructure); + renderSourceProvider.addScannedFiles(files); + } + catch (Exception e) + { + LOGGER.error("Failed to scan and collect cache files for " + levelWrapper + " in " + saveStructure, e); + } + } + + + + //======================// + // Adding scanned files // + //======================// + + /** + * Caller must ensure that this method is called only once, + * and that the {@link FullDataFileHandler} is not used before this method is called. + */ + public static void addScannedFiles( + Collection detectedFiles, boolean useLazyLoading, String fileSuffix, + CreateMetadataFunc createMetadataFunc, + AddUnloadedFileFunc addUnloadedFileFunc, AddLoadedMetaFileFunc addLoadedMetaFileFunc) + { + if (useLazyLoading) + { + lazyAddScannedFile(detectedFiles, fileSuffix, addUnloadedFileFunc); + } + else + { + immediateAddScannedFile(detectedFiles, createMetadataFunc, addLoadedMetaFileFunc); + } + } + private static void lazyAddScannedFile(Collection detectedFiles, String fileSuffix, AddUnloadedFileFunc addUnloadedFileFunc) + { + for (File file : detectedFiles) + { + if (file == null || !file.exists()) + { + // can rarely happen if the user rapidly travels between dimensions + LOGGER.warn("Null or non-existent file: " + ((file != null) ? file.getPath() : "NULL")); + continue; + } + + try + { + DhSectionPos pos = decodePositionFromFileName(file, fileSuffix); + if (pos != null) + { + addUnloadedFileFunc.addFile(pos, file); + } + } + catch (Exception e) + { + LOGGER.error("Failed to read data meta file at " + file + ": ", e); + FileUtil.renameCorruptedFile(file); + } + } + } + private static void immediateAddScannedFile( + Collection detectedFiles, + CreateMetadataFunc createMetadataFunc, AddLoadedMetaFileFunc addLoadedMetaFileFunc) + { + HashMultimap filesByPos = HashMultimap.create(); + { // Sort files by pos. + for (File file : detectedFiles) + { + try + { + AbstractMetaDataContainerFile metaFile = createMetadataFunc.createFile(file); + filesByPos.put(metaFile.pos, metaFile); + } + catch (IOException e) + { + LOGGER.error("Failed to read data meta file at " + file + ": ", e); + FileUtil.renameCorruptedFile(file); + } + } + } + + + // Warn for multiple files with the same pos, and then select the one with the latest timestamp. + for (DhSectionPos pos : filesByPos.keySet()) + { + Collection metaFiles = filesByPos.get(pos); + AbstractMetaDataContainerFile metaFileToUse; + if (metaFiles.size() > 1) + { + // sort by the file's last modified date + metaFileToUse = Collections.max(metaFiles, Comparator.comparingLong(fullDataMetaFile -> fullDataMetaFile.file.lastModified())); + + // log the duplicate files + StringBuilder duplicateMessage = new StringBuilder(); + duplicateMessage.append("Multiple files with the same pos: ").append(pos).append("\n"); + for (AbstractMetaDataContainerFile metaFile : metaFiles) + { + duplicateMessage.append("\t").append(metaFile.file).append("\n"); + } + duplicateMessage.append("\tUsing: ").append(metaFileToUse.file).append("\n"); + duplicateMessage.append("(Other files will be renamed by appending \".old\" to their name.)"); + LOGGER.warn(duplicateMessage.toString()); + + + + // Rename all other files with the same pos to .old + for (AbstractMetaDataContainerFile metaFile : metaFiles) + { + if (metaFile == metaFileToUse) + { + continue; + } + + + File oldFile = new File(metaFile.file + ".old"); + try + { + if (!metaFile.file.renameTo(oldFile)) + { + throw new RuntimeException("Renaming failed"); + } + } + catch (Exception e) + { + LOGGER.error("Failed to rename file: " + metaFile.file + " to " + oldFile, e); + } + } + } + else + { + metaFileToUse = metaFiles.iterator().next(); + } + + // Add file to the list of files. + addLoadedMetaFileFunc.addFile(pos, metaFileToUse); + } + } + + + + //================// + // helper methods // + //================// + + /** @return null if the file name can't be parsed into a {@link DhSectionPos} */ + @Nullable + public static DhSectionPos decodePositionFromFileName(File file, String fileSuffix) + { + String fileName = file.getName(); + if (!fileName.endsWith(fileSuffix)) + { + return null; + } + + fileName = fileName.substring(0, fileName.length() - 4); + return DhSectionPos.deserialize(fileName); + } + + + + //===================// + // helper interfaces // + //===================// + + @FunctionalInterface + public interface CreateMetadataFunc { AbstractMetaDataContainerFile createFile(File file) throws IOException; } + + @FunctionalInterface + public interface AddUnloadedFileFunc { void addFile(DhSectionPos pos, File file); } + @FunctionalInterface + public interface AddLoadedMetaFileFunc { void addFile(DhSectionPos pos, AbstractMetaDataContainerFile metaFile); } + +}