diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index ab63d40f4..e7ba34c06 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -34,7 +34,6 @@ import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.util.MetaFileScanUtil; import com.seibel.distanthorizons.core.util.FileUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; @@ -50,16 +49,13 @@ import java.util.function.Consumer; public class FullDataFileHandler implements IFullDataSourceProvider { - public static final boolean USE_LAZY_LOADING = true; - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); protected static ExecutorService fileHandlerThreadPool; protected static ConfigChangeListener configListener; - private final ConcurrentHashMap unloadedFileBySectionPos = new ConcurrentHashMap<>(); - /** contains the loaded {@link FullDataMetaFile}'s */ - private final ConcurrentHashMap metaFileBySectionPos = new ConcurrentHashMap<>(); + private final ConcurrentHashMap loadedMetaFileBySectionPos = new ConcurrentHashMap<>(); + private final Set missingSectionPos = Collections.newSetFromMap(new ConcurrentHashMap<>()); protected final IDhLevel level; protected final File saveDir; @@ -80,29 +76,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider { LOGGER.warn("Unable to create full data folder, file saving may fail."); } - MetaFileScanUtil.scanFullDataFiles(saveStructure, level.getLevelWrapper(), this); - } - - @Override - public void addScannedFiles(Collection detectedFiles) - { - MetaFileScanUtil.ICreateMetadataFunc createMetadataFunc = (file) -> FullDataMetaFile.createFromExistingFile(this, this.level, file); - - MetaFileScanUtil.IAddUnloadedFileFunc addUnloadedFileFunc = (pos, file) -> - { - this.unloadedFileBySectionPos.put(pos, file); - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - }; - MetaFileScanUtil.IAddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) -> - { - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - this.metaFileBySectionPos.put(pos, (FullDataMetaFile) loadedMetaFile); - }; - - - MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, FullDataMetaFile.FILE_SUFFIX, - createMetadataFunc, - addUnloadedFileFunc, addLoadedMetaFileFunc); } @@ -151,23 +124,25 @@ public class FullDataFileHandler implements IFullDataSourceProvider public FullDataMetaFile getFileIfExist(DhSectionPos pos) { return this.getLoadOrMakeFile(pos, false); } protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile) { - FullDataMetaFile metaFile = this.metaFileBySectionPos.get(pos); + FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos); if (metaFile != null) { return metaFile; } - File fileToLoad = this.unloadedFileBySectionPos.get(pos); // File does exist, but not loaded yet. - if (fileToLoad != null) + File fileToLoad = this.computeDataFilePath(pos); + if (fileToLoad.exists()) { synchronized (this) { + // A file exists, but isn't loaded yet. + // Double check locking for loading file, as loading file means also loading the metadata, which // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. - metaFile = this.metaFileBySectionPos.get(pos); + metaFile = this.loadedMetaFileBySectionPos.get(pos); if (metaFile != null) { return metaFile; // someone else loaded it already. @@ -177,17 +152,17 @@ public class FullDataFileHandler implements IFullDataSourceProvider { metaFile = FullDataMetaFile.createFromExistingFile(this, this.level, fileToLoad); this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - this.metaFileBySectionPos.put(pos, metaFile); + this.loadedMetaFileBySectionPos.put(pos, metaFile); return metaFile; } catch (IOException e) { - LOGGER.error("Failed to read data meta file at " + fileToLoad + ": ", e); + LOGGER.error("Failed to read meta data file at " + fileToLoad + ": ", e); FileUtil.renameCorruptedFile(fileToLoad); } finally { - this.unloadedFileBySectionPos.remove(pos); + this.missingSectionPos.remove(pos); } } } @@ -195,6 +170,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider if (!allowCreateFile) { + this.missingSectionPos.add(pos); return null; } @@ -213,8 +189,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - // This is a CAS with expected null value. - FullDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile); + // This is a Compare And Swap with expected null value. + FullDataMetaFile metaFileCas = this.loadedMetaFileBySectionPos.putIfAbsent(pos, metaFile); + this.missingSectionPos.remove(pos); return metaFileCas == null ? metaFile : metaFileCas; } @@ -254,8 +231,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider continue; } - // check if a file for this pos exists, either loaded and unloaded - if (this.metaFileBySectionPos.containsKey(subPos) || this.unloadedFileBySectionPos.containsKey(subPos)) + // check if a file for this pos is loaded or exists + if (this.loadedMetaFileBySectionPos.containsKey(subPos) || this.computeDataFilePath(subPos).exists()) { allEmpty = false; break outerLoop; @@ -284,14 +261,14 @@ public class FullDataFileHandler implements IFullDataSourceProvider DhSectionPos childPos = pos.getChildByIndex(childIndex); if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos)) { - // load the file if it isn't already - if (this.unloadedFileBySectionPos.containsKey(childPos)) + // get or load the file if necessary + if (!this.loadedMetaFileBySectionPos.containsKey(childPos) && this.computeDataFilePath(childPos).exists()) { this.getLoadOrMakeFile(childPos, true); } - FullDataMetaFile metaFile = this.metaFileBySectionPos.get(childPos); + FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(childPos); if (metaFile != null) { // we have reached a populated leaf node in the quad tree @@ -310,7 +287,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider } } - public void ForEachFile(Consumer consumer) { this.metaFileBySectionPos.values().forEach(consumer); } + public void ForEachFile(Consumer consumer) { this.loadedMetaFileBySectionPos.values().forEach(consumer); } @@ -330,7 +307,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider } private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData) { - FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos); + FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos); if (metaFile != null) { // there is a file for this position @@ -349,7 +326,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider public CompletableFuture flushAndSave() { ArrayList> futures = new ArrayList<>(); - for (FullDataMetaFile metaFile : this.metaFileBySectionPos.values()) + for (FullDataMetaFile metaFile : this.loadedMetaFileBySectionPos.values()) { futures.add(metaFile.flushAndSaveAsync()); } @@ -359,7 +336,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider @Override public CompletableFuture flushAndSave(DhSectionPos sectionPos) { - FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos); + FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos); if (metaFile == null) { return CompletableFuture.completedFuture(null); @@ -497,7 +474,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider FileUtil.renameCorruptedFile(metaFile.file); // remove the FullDataMetaFile since the old one was corrupted - this.metaFileBySectionPos.remove(pos); + this.loadedMetaFileBySectionPos.remove(pos); // create a new FullDataMetaFile to write new data to return this.getLoadOrMakeFile(pos, true); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index b6b242cd8..679b6ed64 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -31,8 +31,6 @@ import java.util.concurrent.ExecutorService; public interface IFullDataSourceProvider extends AutoCloseable { - void addScannedFiles(Collection detectedFiles); - CompletableFuture readAsync(DhSectionPos pos); void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData); CompletableFuture flushAndSave(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java index db6e5fb3c..56cfae961 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java @@ -35,8 +35,6 @@ import java.util.concurrent.CompletableFuture; */ public interface ILodRenderSourceProvider extends AutoCloseable { - void addScannedFiles(Collection detectedFiles); - CompletableFuture readAsync(DhSectionPos pos); void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 9dc6a8bdc..1ed0c2fcb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -27,10 +27,8 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.util.MetaFileScanUtil; import com.seibel.distanthorizons.core.util.FileUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import org.apache.logging.log4j.Logger; import java.io.File; @@ -41,16 +39,12 @@ import java.util.concurrent.atomic.AtomicInteger; public class RenderSourceFileHandler implements ILodRenderSourceProvider { - public static final boolean USE_LAZY_LOADING = true; - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private final ThreadPoolExecutor fileHandlerThreadPool; private final F3Screen.NestedMessage threadPoolMsg; - private final ConcurrentHashMap unloadedFileBySectionPos = new ConcurrentHashMap<>(); - /** contains the loaded {@link RenderDataMetaFile}'s */ - private final ConcurrentHashMap metaFileBySectionPos = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap metaFileBySectionPos = new ConcurrentHashMap<>(); //loadedMetaFileBySectionPos private final IDhClientLevel clientLevel; private final File saveDir; @@ -79,36 +73,6 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log); - - MetaFileScanUtil.scanRenderFiles(saveStructure, clientLevel.getLevelWrapper(), this); - } - - /** - * Caller must ensure that this method is called only once, - * and that the given files are not used before this method is called.

- * - * Used by {@link MetaFileScanUtil#scanRenderFiles(AbstractSaveStructure, ILevelWrapper, ILodRenderSourceProvider)} - */ - @Override - public void addScannedFiles(Collection detectedFiles) - { - MetaFileScanUtil.ICreateMetadataFunc createMetadataFunc = (file) -> RenderDataMetaFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, file); - - MetaFileScanUtil.IAddUnloadedFileFunc addUnloadedFileFunc = (pos, file) -> - { - this.unloadedFileBySectionPos.put(pos, file); - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - }; - MetaFileScanUtil.IAddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) -> - { - this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - this.metaFileBySectionPos.put(pos, (RenderDataMetaFile) loadedMetaFile); - }; - - - MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, RenderDataMetaFile.FILE_SUFFIX, - createMetadataFunc, - addUnloadedFileFunc, addLoadedMetaFileFunc); } @@ -129,6 +93,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); RenderDataMetaFile metaFile = this.getLoadOrMakeFile(pos); if (metaFile == null) { @@ -158,39 +123,26 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider RenderDataMetaFile metaFile = this.metaFileBySectionPos.get(pos); if (metaFile != null) { - // return the loaded file return metaFile; } - // we don't have a loaded file, for that pos, - // do we have an unloaded file for that pos? - File fileToLoad = this.unloadedFileBySectionPos.get(pos); - if (fileToLoad != null && !fileToLoad.exists()) + File fileToLoad = this.computeRenderFilePath(pos); + if (fileToLoad.exists()) { - fileToLoad = null; - this.unloadedFileBySectionPos.remove(pos); - } - - - if (fileToLoad != null) - { - // A file exists, but isn't loaded yet. - - // Double check locking for loading file, as loading file means also loading the metadata, which - // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the - // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. synchronized (this) { - // check if another thread already finished loading this file + // A file exists, but isn't loaded yet. + + // Double check locking for loading file, as loading file means also loading the metadata, which + // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the + // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. metaFile = this.metaFileBySectionPos.get(pos); if (metaFile != null) { - return metaFile; + return metaFile; // someone else loaded it already. } - - // attempt to load the file try { metaFile = RenderDataMetaFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, fileToLoad); @@ -200,41 +152,36 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider } catch (IOException e) { - LOGGER.error("Failed to read render meta file at " + fileToLoad + ": ", e); + LOGGER.error("Failed to read meta data file at " + fileToLoad + ": ", e); FileUtil.renameCorruptedFile(fileToLoad); } - finally - { - this.unloadedFileBySectionPos.remove(pos); - } } } - // Either no file exists for this position - // or the existing file was corrupted. - // Create a new file. + // File does not exist, create it. + // In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS + // to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects. try { - // createFromExistingOrNewFile() is used instead of createFromExistingFile() - // due to a rare issue where the file may already exist but isn't in the file list - metaFile = RenderDataMetaFile.createFromExistingOrNewFile(this.clientLevel, this.fullDataSourceProvider, pos, this.computeRenderFilePath(pos)); - - this.topDetailLevelRef.updateAndGet(newDetailLevel -> Math.max(newDetailLevel, pos.getDetailLevel())); - - // Compare And Swap to handle a concurrency issue where multiple threads created the same Meta File at the same time - RenderDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile); - return (metaFileCas == null) ? metaFile : metaFileCas; + metaFile = RenderDataMetaFile.createNewFileForPos(this.fullDataSourceProvider, this.clientLevel, pos, fileToLoad); } catch (IOException e) { - LOGGER.error("IOException on creating new data file at "+pos, e); + LOGGER.error("IOException on creating new render data file at "+pos, e); return null; } + + this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); + + // This is a Compare And Swap with expected null value. + RenderDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile); + return (metaFileCas == null) ? metaFile : metaFileCas; } + //=============// // data saving // //=============// @@ -282,16 +229,13 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider @Override public CompletableFuture flushAndSaveAsync() { - LOGGER.info("Shutting down " + RenderSourceFileHandler.class.getSimpleName() + "..."); - ArrayList> futures = new ArrayList<>(); for (RenderDataMetaFile metaFile : this.metaFileBySectionPos.values()) { futures.add(metaFile.flushAndSaveAsync()); } - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .whenComplete((voidObj, exception) -> LOGGER.info("Finished saving " + RenderSourceFileHandler.class.getSimpleName())); + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } @@ -305,7 +249,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider { ArrayList lines = new ArrayList<>(); lines.add("Render Source File Handler [" + this.clientLevel.getClientLevelWrapper().getDimensionType().getDimensionName() + "]"); - lines.add(" Loaded files: " + this.metaFileBySectionPos.size() + " / " + (this.unloadedFileBySectionPos.size() + this.metaFileBySectionPos.size())); + lines.add(" Loaded files: " + this.metaFileBySectionPos.size()); lines.add(" Thread pool tasks: " + this.fileHandlerThreadPool.getQueue().size() + " (completed: " + this.fileHandlerThreadPool.getCompletedTaskCount() + ")"); int totalFutures = this.taskTracker.size(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/MetaFileScanUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/MetaFileScanUtil.java deleted file mode 100644 index 726c92d0a..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/MetaFileScanUtil.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.util; - -import com.google.common.collect.HashMultimap; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile; -import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider; -import com.seibel.distanthorizons.core.file.renderfile.RenderDataMetaFile; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** Used to pull in the initial files used by both {@link IFullDataSourceProvider} and {@link ILodRenderSourceProvider}s. */ -public class MetaFileScanUtil -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - public static final int MAX_SCAN_DEPTH = 5; - - - - //===============// - // file scanning // - //===============// - - // file scanning means to find all File's in a given directory - - // TODO merge with the below method - public static void scanFullDataFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, IFullDataSourceProvider dataSourceProvider) - { - try (Stream pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) - { - List files = pathStream.filter( - path -> path.toFile().getName().endsWith(FullDataMetaFile.FILE_SUFFIX) && path.toFile().isFile() - ).map(Path::toFile).collect(Collectors.toList()); - LOGGER.info("Found " + files.size() + " full data files for " + levelWrapper + " in " + saveStructure); - dataSourceProvider.addScannedFiles(files); - } - catch (Exception e) - { - LOGGER.error("Failed to scan and collect full data files for " + levelWrapper + " in " + saveStructure, e); - } - } - - // TODO merge with the above method - public static void scanRenderFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, ILodRenderSourceProvider renderSourceProvider) - { - try (Stream pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) - { - List files = pathStream.filter( - path -> path.toFile().getName().endsWith(RenderDataMetaFile.FILE_SUFFIX) && path.toFile().isFile() - ).map(Path::toFile).collect(Collectors.toList()); - LOGGER.info("Found " + files.size() + " render cache files for " + levelWrapper + " in " + saveStructure); - renderSourceProvider.addScannedFiles(files); - } - catch (Exception e) - { - LOGGER.error("Failed to scan and collect cache files for " + levelWrapper + " in " + saveStructure, e); - } - } - - - - //======================// - // Adding scanned files // - //======================// - - /** - * Caller must ensure that this method is called only once, - * and that the {@link FullDataFileHandler} is not used before this method is called. - */ - public static void addScannedFiles( - Collection detectedFiles, boolean useLazyLoading, String fileSuffix, - ICreateMetadataFunc createMetadataFunc, - IAddUnloadedFileFunc addUnloadedFileFunc, IAddLoadedMetaFileFunc addLoadedMetaFileFunc) - { - if (useLazyLoading) - { - lazyAddScannedFile(detectedFiles, fileSuffix, addUnloadedFileFunc); - } - else - { - immediateAddScannedFile(detectedFiles, createMetadataFunc, addLoadedMetaFileFunc); - } - } - private static void lazyAddScannedFile(Collection detectedFiles, String fileSuffix, IAddUnloadedFileFunc addUnloadedFileFunc) - { - for (File file : detectedFiles) - { - if (file == null || !file.exists()) - { - // can rarely happen if the user rapidly travels between dimensions - LOGGER.warn("Null or non-existent file: " + ((file != null) ? file.getPath() : "NULL")); - continue; - } - - try - { - DhSectionPos pos = decodePositionFromFileName(file, fileSuffix); - if (pos != null) - { - addUnloadedFileFunc.addFile(pos, file); - } - } - catch (Exception e) - { - LOGGER.error("Failed to read data meta file at " + file + ": ", e); - FileUtil.renameCorruptedFile(file); - } - } - } - private static void immediateAddScannedFile( - Collection detectedFiles, - ICreateMetadataFunc createMetadataFunc, IAddLoadedMetaFileFunc addLoadedMetaFileFunc) - { - HashMultimap filesByPos = HashMultimap.create(); - { // Sort files by pos. - for (File file : detectedFiles) - { - try - { - AbstractMetaDataContainerFile metaFile = createMetadataFunc.createFile(file); - filesByPos.put(metaFile.pos, metaFile); - } - catch (IOException e) - { - LOGGER.error("Failed to read data meta file at " + file + ": ", e); - FileUtil.renameCorruptedFile(file); - } - } - } - - - // Warn for multiple files with the same pos, and then select the one with the latest timestamp. - for (DhSectionPos pos : filesByPos.keySet()) - { - Collection metaFiles = filesByPos.get(pos); - AbstractMetaDataContainerFile metaFileToUse; - if (metaFiles.size() > 1) - { - // sort by the file's last modified date - metaFileToUse = Collections.max(metaFiles, Comparator.comparingLong(fullDataMetaFile -> fullDataMetaFile.file.lastModified())); - - // log the duplicate files - StringBuilder duplicateMessage = new StringBuilder(); - duplicateMessage.append("Multiple files with the same pos: ").append(pos).append("\n"); - for (AbstractMetaDataContainerFile metaFile : metaFiles) - { - duplicateMessage.append("\t").append(metaFile.file).append("\n"); - } - duplicateMessage.append("\tUsing: ").append(metaFileToUse.file).append("\n"); - duplicateMessage.append("(Other files will be renamed by appending \".old\" to their name.)"); - LOGGER.warn(duplicateMessage.toString()); - - - - // Rename all other files with the same pos to .old - for (AbstractMetaDataContainerFile metaFile : metaFiles) - { - if (metaFile == metaFileToUse) - { - continue; - } - - - File oldFile = new File(metaFile.file + ".old"); - try - { - if (!metaFile.file.renameTo(oldFile)) - { - throw new RuntimeException("Renaming failed"); - } - } - catch (Exception e) - { - LOGGER.error("Failed to rename file: " + metaFile.file + " to " + oldFile, e); - } - } - } - else - { - metaFileToUse = metaFiles.iterator().next(); - } - - // Add file to the list of files. - addLoadedMetaFileFunc.addFile(pos, metaFileToUse); - } - } - - - - //================// - // helper methods // - //================// - - /** @return null if the file name can't be parsed into a {@link DhSectionPos} */ - @Nullable - public static DhSectionPos decodePositionFromFileName(File file, String fileSuffix) - { - String fileName = file.getName(); - if (!fileName.endsWith(fileSuffix)) - { - return null; - } - - fileName = fileName.substring(0, fileName.length() - 4); - return DhSectionPos.deserialize(fileName); - } - - - - //===================// - // helper interfaces // - //===================// - - @FunctionalInterface - public interface ICreateMetadataFunc { AbstractMetaDataContainerFile createFile(File file) throws IOException; } - - @FunctionalInterface - public interface IAddUnloadedFileFunc { void addFile(DhSectionPos pos, File file); } - @FunctionalInterface - public interface IAddLoadedMetaFileFunc { void addFile(DhSectionPos pos, AbstractMetaDataContainerFile metaFile); } - -}