Improve initial level loading time
This commit is contained in:
James Seibel
+25
-48
@@ -34,7 +34,6 @@ import com.seibel.distanthorizons.core.pos.DhLodPos;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
|
||||
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
|
||||
import com.seibel.distanthorizons.core.util.MetaFileScanUtil;
|
||||
import com.seibel.distanthorizons.core.util.FileUtil;
|
||||
import com.seibel.distanthorizons.core.util.LodUtil;
|
||||
import com.seibel.distanthorizons.core.util.ThreadUtil;
|
||||
@@ -50,16 +49,13 @@ import java.util.function.Consumer;
|
||||
|
||||
public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
{
|
||||
public static final boolean USE_LAZY_LOADING = true;
|
||||
|
||||
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
|
||||
|
||||
protected static ExecutorService fileHandlerThreadPool;
|
||||
protected static ConfigChangeListener<Integer> configListener;
|
||||
|
||||
private final ConcurrentHashMap<DhSectionPos, File> unloadedFileBySectionPos = new ConcurrentHashMap<>();
|
||||
/** contains the loaded {@link FullDataMetaFile}'s */
|
||||
private final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> metaFileBySectionPos = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> loadedMetaFileBySectionPos = new ConcurrentHashMap<>();
|
||||
private final Set<DhSectionPos> missingSectionPos = Collections.newSetFromMap(new ConcurrentHashMap<>());
|
||||
|
||||
protected final IDhLevel level;
|
||||
protected final File saveDir;
|
||||
@@ -80,29 +76,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
{
|
||||
LOGGER.warn("Unable to create full data folder, file saving may fail.");
|
||||
}
|
||||
MetaFileScanUtil.scanFullDataFiles(saveStructure, level.getLevelWrapper(), this);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addScannedFiles(Collection<File> detectedFiles)
|
||||
{
|
||||
MetaFileScanUtil.ICreateMetadataFunc createMetadataFunc = (file) -> FullDataMetaFile.createFromExistingFile(this, this.level, file);
|
||||
|
||||
MetaFileScanUtil.IAddUnloadedFileFunc addUnloadedFileFunc = (pos, file) ->
|
||||
{
|
||||
this.unloadedFileBySectionPos.put(pos, file);
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
};
|
||||
MetaFileScanUtil.IAddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) ->
|
||||
{
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
this.metaFileBySectionPos.put(pos, (FullDataMetaFile) loadedMetaFile);
|
||||
};
|
||||
|
||||
|
||||
MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, FullDataMetaFile.FILE_SUFFIX,
|
||||
createMetadataFunc,
|
||||
addUnloadedFileFunc, addLoadedMetaFileFunc);
|
||||
}
|
||||
|
||||
|
||||
@@ -151,23 +124,25 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
public FullDataMetaFile getFileIfExist(DhSectionPos pos) { return this.getLoadOrMakeFile(pos, false); }
|
||||
protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile)
|
||||
{
|
||||
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(pos);
|
||||
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(pos);
|
||||
if (metaFile != null)
|
||||
{
|
||||
return metaFile;
|
||||
}
|
||||
|
||||
|
||||
File fileToLoad = this.unloadedFileBySectionPos.get(pos);
|
||||
// File does exist, but not loaded yet.
|
||||
if (fileToLoad != null)
|
||||
File fileToLoad = this.computeDataFilePath(pos);
|
||||
if (fileToLoad.exists())
|
||||
{
|
||||
synchronized (this)
|
||||
{
|
||||
// A file exists, but isn't loaded yet.
|
||||
|
||||
// Double check locking for loading file, as loading file means also loading the metadata, which
|
||||
// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
|
||||
// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
|
||||
metaFile = this.metaFileBySectionPos.get(pos);
|
||||
metaFile = this.loadedMetaFileBySectionPos.get(pos);
|
||||
if (metaFile != null)
|
||||
{
|
||||
return metaFile; // someone else loaded it already.
|
||||
@@ -177,17 +152,17 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
{
|
||||
metaFile = FullDataMetaFile.createFromExistingFile(this, this.level, fileToLoad);
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
this.metaFileBySectionPos.put(pos, metaFile);
|
||||
this.loadedMetaFileBySectionPos.put(pos, metaFile);
|
||||
return metaFile;
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
LOGGER.error("Failed to read data meta file at " + fileToLoad + ": ", e);
|
||||
LOGGER.error("Failed to read meta data file at " + fileToLoad + ": ", e);
|
||||
FileUtil.renameCorruptedFile(fileToLoad);
|
||||
}
|
||||
finally
|
||||
{
|
||||
this.unloadedFileBySectionPos.remove(pos);
|
||||
this.missingSectionPos.remove(pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,6 +170,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
|
||||
if (!allowCreateFile)
|
||||
{
|
||||
this.missingSectionPos.add(pos);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -213,8 +189,9 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
|
||||
// This is a CAS with expected null value.
|
||||
FullDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile);
|
||||
// This is a Compare And Swap with expected null value.
|
||||
FullDataMetaFile metaFileCas = this.loadedMetaFileBySectionPos.putIfAbsent(pos, metaFile);
|
||||
this.missingSectionPos.remove(pos);
|
||||
return metaFileCas == null ? metaFile : metaFileCas;
|
||||
}
|
||||
|
||||
@@ -254,8 +231,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
continue;
|
||||
}
|
||||
|
||||
// check if a file for this pos exists, either loaded and unloaded
|
||||
if (this.metaFileBySectionPos.containsKey(subPos) || this.unloadedFileBySectionPos.containsKey(subPos))
|
||||
// check if a file for this pos is loaded or exists
|
||||
if (this.loadedMetaFileBySectionPos.containsKey(subPos) || this.computeDataFilePath(subPos).exists())
|
||||
{
|
||||
allEmpty = false;
|
||||
break outerLoop;
|
||||
@@ -284,14 +261,14 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
DhSectionPos childPos = pos.getChildByIndex(childIndex);
|
||||
if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos))
|
||||
{
|
||||
// load the file if it isn't already
|
||||
if (this.unloadedFileBySectionPos.containsKey(childPos))
|
||||
// get or load the file if necessary
|
||||
if (!this.loadedMetaFileBySectionPos.containsKey(childPos) && this.computeDataFilePath(childPos).exists())
|
||||
{
|
||||
this.getLoadOrMakeFile(childPos, true);
|
||||
}
|
||||
|
||||
|
||||
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(childPos);
|
||||
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(childPos);
|
||||
if (metaFile != null)
|
||||
{
|
||||
// we have reached a populated leaf node in the quad tree
|
||||
@@ -310,7 +287,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
}
|
||||
}
|
||||
|
||||
public void ForEachFile(Consumer<FullDataMetaFile> consumer) { this.metaFileBySectionPos.values().forEach(consumer); }
|
||||
public void ForEachFile(Consumer<FullDataMetaFile> consumer) { this.loadedMetaFileBySectionPos.values().forEach(consumer); }
|
||||
|
||||
|
||||
|
||||
@@ -330,7 +307,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
}
|
||||
private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData)
|
||||
{
|
||||
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos);
|
||||
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos);
|
||||
if (metaFile != null)
|
||||
{
|
||||
// there is a file for this position
|
||||
@@ -349,7 +326,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
public CompletableFuture<Void> flushAndSave()
|
||||
{
|
||||
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
|
||||
for (FullDataMetaFile metaFile : this.metaFileBySectionPos.values())
|
||||
for (FullDataMetaFile metaFile : this.loadedMetaFileBySectionPos.values())
|
||||
{
|
||||
futures.add(metaFile.flushAndSaveAsync());
|
||||
}
|
||||
@@ -359,7 +336,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
@Override
|
||||
public CompletableFuture<Void> flushAndSave(DhSectionPos sectionPos)
|
||||
{
|
||||
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos);
|
||||
FullDataMetaFile metaFile = this.loadedMetaFileBySectionPos.get(sectionPos);
|
||||
if (metaFile == null)
|
||||
{
|
||||
return CompletableFuture.completedFuture(null);
|
||||
@@ -497,7 +474,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
|
||||
|
||||
FileUtil.renameCorruptedFile(metaFile.file);
|
||||
// remove the FullDataMetaFile since the old one was corrupted
|
||||
this.metaFileBySectionPos.remove(pos);
|
||||
this.loadedMetaFileBySectionPos.remove(pos);
|
||||
// create a new FullDataMetaFile to write new data to
|
||||
return this.getLoadOrMakeFile(pos, true);
|
||||
}
|
||||
|
||||
-2
@@ -31,8 +31,6 @@ import java.util.concurrent.ExecutorService;
|
||||
|
||||
public interface IFullDataSourceProvider extends AutoCloseable
|
||||
{
|
||||
void addScannedFiles(Collection<File> detectedFiles);
|
||||
|
||||
CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos);
|
||||
void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData);
|
||||
CompletableFuture<Void> flushAndSave();
|
||||
|
||||
-2
@@ -35,8 +35,6 @@ import java.util.concurrent.CompletableFuture;
|
||||
*/
|
||||
public interface ILodRenderSourceProvider extends AutoCloseable
|
||||
{
|
||||
void addScannedFiles(Collection<File> detectedFiles);
|
||||
|
||||
CompletableFuture<ColumnRenderSource> readAsync(DhSectionPos pos);
|
||||
|
||||
void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData);
|
||||
|
||||
+25
-81
@@ -27,10 +27,8 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
|
||||
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
|
||||
import com.seibel.distanthorizons.core.level.IDhClientLevel;
|
||||
import com.seibel.distanthorizons.core.util.MetaFileScanUtil;
|
||||
import com.seibel.distanthorizons.core.util.FileUtil;
|
||||
import com.seibel.distanthorizons.core.util.ThreadUtil;
|
||||
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
@@ -41,16 +39,12 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class RenderSourceFileHandler implements ILodRenderSourceProvider
|
||||
{
|
||||
public static final boolean USE_LAZY_LOADING = true;
|
||||
|
||||
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
|
||||
|
||||
private final ThreadPoolExecutor fileHandlerThreadPool;
|
||||
private final F3Screen.NestedMessage threadPoolMsg;
|
||||
|
||||
private final ConcurrentHashMap<DhSectionPos, File> unloadedFileBySectionPos = new ConcurrentHashMap<>();
|
||||
/** contains the loaded {@link RenderDataMetaFile}'s */
|
||||
private final ConcurrentHashMap<DhSectionPos, RenderDataMetaFile> metaFileBySectionPos = new ConcurrentHashMap<>();
|
||||
protected final ConcurrentHashMap<DhSectionPos, RenderDataMetaFile> metaFileBySectionPos = new ConcurrentHashMap<>(); //loadedMetaFileBySectionPos
|
||||
|
||||
private final IDhClientLevel clientLevel;
|
||||
private final File saveDir;
|
||||
@@ -79,36 +73,6 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
|
||||
|
||||
|
||||
this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log);
|
||||
|
||||
MetaFileScanUtil.scanRenderFiles(saveStructure, clientLevel.getLevelWrapper(), this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Caller must ensure that this method is called only once,
|
||||
* and that the given files are not used before this method is called. <br><br>
|
||||
*
|
||||
* Used by {@link MetaFileScanUtil#scanRenderFiles(AbstractSaveStructure, ILevelWrapper, ILodRenderSourceProvider)}
|
||||
*/
|
||||
@Override
|
||||
public void addScannedFiles(Collection<File> detectedFiles)
|
||||
{
|
||||
MetaFileScanUtil.ICreateMetadataFunc createMetadataFunc = (file) -> RenderDataMetaFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, file);
|
||||
|
||||
MetaFileScanUtil.IAddUnloadedFileFunc addUnloadedFileFunc = (pos, file) ->
|
||||
{
|
||||
this.unloadedFileBySectionPos.put(pos, file);
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
};
|
||||
MetaFileScanUtil.IAddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) ->
|
||||
{
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
this.metaFileBySectionPos.put(pos, (RenderDataMetaFile) loadedMetaFile);
|
||||
};
|
||||
|
||||
|
||||
MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, RenderDataMetaFile.FILE_SUFFIX,
|
||||
createMetadataFunc,
|
||||
addUnloadedFileFunc, addLoadedMetaFileFunc);
|
||||
}
|
||||
|
||||
|
||||
@@ -129,6 +93,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
|
||||
|
||||
|
||||
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
RenderDataMetaFile metaFile = this.getLoadOrMakeFile(pos);
|
||||
if (metaFile == null)
|
||||
{
|
||||
@@ -158,39 +123,26 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
|
||||
RenderDataMetaFile metaFile = this.metaFileBySectionPos.get(pos);
|
||||
if (metaFile != null)
|
||||
{
|
||||
// return the loaded file
|
||||
return metaFile;
|
||||
}
|
||||
|
||||
|
||||
// we don't have a loaded file, for that pos,
|
||||
// do we have an unloaded file for that pos?
|
||||
File fileToLoad = this.unloadedFileBySectionPos.get(pos);
|
||||
if (fileToLoad != null && !fileToLoad.exists())
|
||||
File fileToLoad = this.computeRenderFilePath(pos);
|
||||
if (fileToLoad.exists())
|
||||
{
|
||||
fileToLoad = null;
|
||||
this.unloadedFileBySectionPos.remove(pos);
|
||||
}
|
||||
|
||||
|
||||
if (fileToLoad != null)
|
||||
{
|
||||
// A file exists, but isn't loaded yet.
|
||||
|
||||
// Double check locking for loading file, as loading file means also loading the metadata, which
|
||||
// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
|
||||
// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
|
||||
synchronized (this)
|
||||
{
|
||||
// check if another thread already finished loading this file
|
||||
// A file exists, but isn't loaded yet.
|
||||
|
||||
// Double check locking for loading file, as loading file means also loading the metadata, which
|
||||
// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
|
||||
// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
|
||||
metaFile = this.metaFileBySectionPos.get(pos);
|
||||
if (metaFile != null)
|
||||
{
|
||||
return metaFile;
|
||||
return metaFile; // someone else loaded it already.
|
||||
}
|
||||
|
||||
|
||||
// attempt to load the file
|
||||
try
|
||||
{
|
||||
metaFile = RenderDataMetaFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, fileToLoad);
|
||||
@@ -200,41 +152,36 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
LOGGER.error("Failed to read render meta file at " + fileToLoad + ": ", e);
|
||||
LOGGER.error("Failed to read meta data file at " + fileToLoad + ": ", e);
|
||||
FileUtil.renameCorruptedFile(fileToLoad);
|
||||
}
|
||||
finally
|
||||
{
|
||||
this.unloadedFileBySectionPos.remove(pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Either no file exists for this position
|
||||
// or the existing file was corrupted.
|
||||
// Create a new file.
|
||||
// File does not exist, create it.
|
||||
// In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS
|
||||
// to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects.
|
||||
try
|
||||
{
|
||||
// createFromExistingOrNewFile() is used instead of createFromExistingFile()
|
||||
// due to a rare issue where the file may already exist but isn't in the file list
|
||||
metaFile = RenderDataMetaFile.createFromExistingOrNewFile(this.clientLevel, this.fullDataSourceProvider, pos, this.computeRenderFilePath(pos));
|
||||
|
||||
this.topDetailLevelRef.updateAndGet(newDetailLevel -> Math.max(newDetailLevel, pos.getDetailLevel()));
|
||||
|
||||
// Compare And Swap to handle a concurrency issue where multiple threads created the same Meta File at the same time
|
||||
RenderDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile);
|
||||
return (metaFileCas == null) ? metaFile : metaFileCas;
|
||||
metaFile = RenderDataMetaFile.createNewFileForPos(this.fullDataSourceProvider, this.clientLevel, pos, fileToLoad);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
LOGGER.error("IOException on creating new data file at "+pos, e);
|
||||
LOGGER.error("IOException on creating new render data file at "+pos, e);
|
||||
return null;
|
||||
}
|
||||
|
||||
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
|
||||
|
||||
// This is a Compare And Swap with expected null value.
|
||||
RenderDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile);
|
||||
return (metaFileCas == null) ? metaFile : metaFileCas;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
//=============//
|
||||
// data saving //
|
||||
//=============//
|
||||
@@ -282,16 +229,13 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
|
||||
@Override
|
||||
public CompletableFuture<Void> flushAndSaveAsync()
|
||||
{
|
||||
LOGGER.info("Shutting down " + RenderSourceFileHandler.class.getSimpleName() + "...");
|
||||
|
||||
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
|
||||
for (RenderDataMetaFile metaFile : this.metaFileBySectionPos.values())
|
||||
{
|
||||
futures.add(metaFile.flushAndSaveAsync());
|
||||
}
|
||||
|
||||
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
|
||||
.whenComplete((voidObj, exception) -> LOGGER.info("Finished saving " + RenderSourceFileHandler.class.getSimpleName()));
|
||||
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
|
||||
}
|
||||
|
||||
|
||||
@@ -305,7 +249,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
|
||||
{
|
||||
ArrayList<String> lines = new ArrayList<>();
|
||||
lines.add("Render Source File Handler [" + this.clientLevel.getClientLevelWrapper().getDimensionType().getDimensionName() + "]");
|
||||
lines.add(" Loaded files: " + this.metaFileBySectionPos.size() + " / " + (this.unloadedFileBySectionPos.size() + this.metaFileBySectionPos.size()));
|
||||
lines.add(" Loaded files: " + this.metaFileBySectionPos.size());
|
||||
lines.add(" Thread pool tasks: " + this.fileHandlerThreadPool.getQueue().size() + " (completed: " + this.fileHandlerThreadPool.getCompletedTaskCount() + ")");
|
||||
|
||||
int totalFutures = this.taskTracker.size();
|
||||
|
||||
@@ -1,257 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Distant Horizons mod
|
||||
* licensed under the GNU LGPL v3 License.
|
||||
*
|
||||
* Copyright (C) 2020-2023 James Seibel
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.seibel.distanthorizons.core.util;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandler;
|
||||
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
|
||||
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
|
||||
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
|
||||
import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider;
|
||||
import com.seibel.distanthorizons.core.file.renderfile.RenderDataMetaFile;
|
||||
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
|
||||
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/** Used to pull in the initial files used by both {@link IFullDataSourceProvider} and {@link ILodRenderSourceProvider}s. */
|
||||
public class MetaFileScanUtil
|
||||
{
|
||||
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
|
||||
public static final int MAX_SCAN_DEPTH = 5;
|
||||
|
||||
|
||||
|
||||
//===============//
|
||||
// file scanning //
|
||||
//===============//
|
||||
|
||||
// file scanning means to find all File's in a given directory
|
||||
|
||||
// TODO merge with the below method
|
||||
public static void scanFullDataFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, IFullDataSourceProvider dataSourceProvider)
|
||||
{
|
||||
try (Stream<Path> pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
|
||||
{
|
||||
List<File> files = pathStream.filter(
|
||||
path -> path.toFile().getName().endsWith(FullDataMetaFile.FILE_SUFFIX) && path.toFile().isFile()
|
||||
).map(Path::toFile).collect(Collectors.toList());
|
||||
LOGGER.info("Found " + files.size() + " full data files for " + levelWrapper + " in " + saveStructure);
|
||||
dataSourceProvider.addScannedFiles(files);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOGGER.error("Failed to scan and collect full data files for " + levelWrapper + " in " + saveStructure, e);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO merge with the above method
|
||||
public static void scanRenderFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, ILodRenderSourceProvider renderSourceProvider)
|
||||
{
|
||||
try (Stream<Path> pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
|
||||
{
|
||||
List<File> files = pathStream.filter(
|
||||
path -> path.toFile().getName().endsWith(RenderDataMetaFile.FILE_SUFFIX) && path.toFile().isFile()
|
||||
).map(Path::toFile).collect(Collectors.toList());
|
||||
LOGGER.info("Found " + files.size() + " render cache files for " + levelWrapper + " in " + saveStructure);
|
||||
renderSourceProvider.addScannedFiles(files);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOGGER.error("Failed to scan and collect cache files for " + levelWrapper + " in " + saveStructure, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
//======================//
|
||||
// Adding scanned files //
|
||||
//======================//
|
||||
|
||||
/**
|
||||
* Caller must ensure that this method is called only once,
|
||||
* and that the {@link FullDataFileHandler} is not used before this method is called.
|
||||
*/
|
||||
public static void addScannedFiles(
|
||||
Collection<File> detectedFiles, boolean useLazyLoading, String fileSuffix,
|
||||
ICreateMetadataFunc createMetadataFunc,
|
||||
IAddUnloadedFileFunc addUnloadedFileFunc, IAddLoadedMetaFileFunc addLoadedMetaFileFunc)
|
||||
{
|
||||
if (useLazyLoading)
|
||||
{
|
||||
lazyAddScannedFile(detectedFiles, fileSuffix, addUnloadedFileFunc);
|
||||
}
|
||||
else
|
||||
{
|
||||
immediateAddScannedFile(detectedFiles, createMetadataFunc, addLoadedMetaFileFunc);
|
||||
}
|
||||
}
|
||||
private static void lazyAddScannedFile(Collection<File> detectedFiles, String fileSuffix, IAddUnloadedFileFunc addUnloadedFileFunc)
|
||||
{
|
||||
for (File file : detectedFiles)
|
||||
{
|
||||
if (file == null || !file.exists())
|
||||
{
|
||||
// can rarely happen if the user rapidly travels between dimensions
|
||||
LOGGER.warn("Null or non-existent file: " + ((file != null) ? file.getPath() : "NULL"));
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
DhSectionPos pos = decodePositionFromFileName(file, fileSuffix);
|
||||
if (pos != null)
|
||||
{
|
||||
addUnloadedFileFunc.addFile(pos, file);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOGGER.error("Failed to read data meta file at " + file + ": ", e);
|
||||
FileUtil.renameCorruptedFile(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
private static void immediateAddScannedFile(
|
||||
Collection<File> detectedFiles,
|
||||
ICreateMetadataFunc createMetadataFunc, IAddLoadedMetaFileFunc addLoadedMetaFileFunc)
|
||||
{
|
||||
HashMultimap<DhSectionPos, AbstractMetaDataContainerFile> filesByPos = HashMultimap.create();
|
||||
{ // Sort files by pos.
|
||||
for (File file : detectedFiles)
|
||||
{
|
||||
try
|
||||
{
|
||||
AbstractMetaDataContainerFile metaFile = createMetadataFunc.createFile(file);
|
||||
filesByPos.put(metaFile.pos, metaFile);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
LOGGER.error("Failed to read data meta file at " + file + ": ", e);
|
||||
FileUtil.renameCorruptedFile(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Warn for multiple files with the same pos, and then select the one with the latest timestamp.
|
||||
for (DhSectionPos pos : filesByPos.keySet())
|
||||
{
|
||||
Collection<AbstractMetaDataContainerFile> metaFiles = filesByPos.get(pos);
|
||||
AbstractMetaDataContainerFile metaFileToUse;
|
||||
if (metaFiles.size() > 1)
|
||||
{
|
||||
// sort by the file's last modified date
|
||||
metaFileToUse = Collections.max(metaFiles, Comparator.comparingLong(fullDataMetaFile -> fullDataMetaFile.file.lastModified()));
|
||||
|
||||
// log the duplicate files
|
||||
StringBuilder duplicateMessage = new StringBuilder();
|
||||
duplicateMessage.append("Multiple files with the same pos: ").append(pos).append("\n");
|
||||
for (AbstractMetaDataContainerFile metaFile : metaFiles)
|
||||
{
|
||||
duplicateMessage.append("\t").append(metaFile.file).append("\n");
|
||||
}
|
||||
duplicateMessage.append("\tUsing: ").append(metaFileToUse.file).append("\n");
|
||||
duplicateMessage.append("(Other files will be renamed by appending \".old\" to their name.)");
|
||||
LOGGER.warn(duplicateMessage.toString());
|
||||
|
||||
|
||||
|
||||
// Rename all other files with the same pos to .old
|
||||
for (AbstractMetaDataContainerFile metaFile : metaFiles)
|
||||
{
|
||||
if (metaFile == metaFileToUse)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
File oldFile = new File(metaFile.file + ".old");
|
||||
try
|
||||
{
|
||||
if (!metaFile.file.renameTo(oldFile))
|
||||
{
|
||||
throw new RuntimeException("Renaming failed");
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOGGER.error("Failed to rename file: " + metaFile.file + " to " + oldFile, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
metaFileToUse = metaFiles.iterator().next();
|
||||
}
|
||||
|
||||
// Add file to the list of files.
|
||||
addLoadedMetaFileFunc.addFile(pos, metaFileToUse);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
//================//
|
||||
// helper methods //
|
||||
//================//
|
||||
|
||||
/** @return null if the file name can't be parsed into a {@link DhSectionPos} */
|
||||
@Nullable
|
||||
public static DhSectionPos decodePositionFromFileName(File file, String fileSuffix)
|
||||
{
|
||||
String fileName = file.getName();
|
||||
if (!fileName.endsWith(fileSuffix))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
fileName = fileName.substring(0, fileName.length() - 4);
|
||||
return DhSectionPos.deserialize(fileName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
//===================//
|
||||
// helper interfaces //
|
||||
//===================//
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ICreateMetadataFunc { AbstractMetaDataContainerFile createFile(File file) throws IOException; }
|
||||
|
||||
@FunctionalInterface
|
||||
public interface IAddUnloadedFileFunc { void addFile(DhSectionPos pos, File file); }
|
||||
@FunctionalInterface
|
||||
public interface IAddLoadedMetaFileFunc { void addFile(DhSectionPos pos, AbstractMetaDataContainerFile metaFile); }
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user