wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move meta file scanning into FileScanUtil (now MetaFileScanUtil)

Browse Source
This commit is contained in:
James Seibel
2023-09-04 09:46:32 -05:00
parent f21545a3af
commit 08359b1b31
9 changed files with 387 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java
+99 -200
View File
@@ -19,7 +19,6 @@
package com.seibel.distanthorizons.core.file.fullDatafile;
import com.google.common.collect.HashMultimap;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
@@ -33,12 +32,11 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.distanthorizons.core.util.FileScanUtil;
import com.seibel.distanthorizons.core.util.MetaFileScanUtil;
import com.seibel.distanthorizons.core.util.FileUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
@@ -48,8 +46,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.Function;
import static com.seibel.distanthorizons.core.util.FileScanUtil.LOD_FILE_POSTFIX;
public class FullDataFileHandler implements IFullDataSourceProvider
{
public static final boolean USE_LAZY_LOADING = true;
@@ -59,18 +55,17 @@ public class FullDataFileHandler implements IFullDataSourceProvider
protected static ExecutorService fileHandlerThreadPool;
protected static ConfigChangeListener<Integer> configListener;
private final ConcurrentHashMap<DhSectionPos, File> unloadedFiles = new ConcurrentHashMap<>();
private final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> fileBySectionPos = new ConcurrentHashMap<>();
public void ForEachFile(Consumer<FullDataMetaFile> consumer) { this.fileBySectionPos.values().forEach(consumer); }
private LinkedList<Consumer<IFullDataSource>> onUpdatedListeners = new LinkedList<>();
private final ConcurrentHashMap<DhSectionPos, File> unloadedFileBySectionPos = new ConcurrentHashMap<>();
/** contains the loaded {@link FullDataMetaFile}'s */
private final ConcurrentHashMap<DhSectionPos, FullDataMetaFile> metaFileBySectionPos = new ConcurrentHashMap<>();
protected final IDhLevel level;
protected final File saveDir;
protected final AtomicInteger topDetailLevel = new AtomicInteger(0);
protected final AtomicInteger topDetailLevelRef = new AtomicInteger(0);
protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET;
//=============//
// constructor //
//=============//
@@ -83,133 +78,85 @@ public class FullDataFileHandler implements IFullDataSourceProvider
{
LOGGER.warn("Unable to create full data folder, file saving may fail.");
}
FileScanUtil.scanFullDataFiles(saveStructure, level.getLevelWrapper(), this);
MetaFileScanUtil.scanFullDataFiles(saveStructure, level.getLevelWrapper(), this);
}
// constructor helpers //
@Override
public void addScannedFiles(Collection<File> detectedFiles)
{
MetaFileScanUtil.CreateMetadataFunc createMetadataFunc = (file) -> new FullDataMetaFile(this, this.level, file);
MetaFileScanUtil.AddUnloadedFileFunc addUnloadedFileFunc = (pos, file) ->
{
this.unloadedFileBySectionPos.put(pos, file);
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
};
MetaFileScanUtil.AddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) ->
{
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
this.metaFileBySectionPos.put(pos, (FullDataMetaFile) loadedMetaFile);
};
MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, FullDataMetaFile.FILE_SUFFIX,
createMetadataFunc,
addUnloadedFileFunc, addLoadedMetaFileFunc);
}
//===============//
// file handling //
//===============//
/**
* Caller must ensure that this method is called only once,
* and that the {@link FullDataFileHandler} is not used before this method is called.
* Returns the {@link IFullDataSource} for the given section position. <Br>
* The returned data source may be null. <Br> <Br>
*
* For now, if result is null, it prob means error has occurred when loading or creating the file object. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
@Override
public void addScannedFile(Collection<File> detectedFiles)
public CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos)
{
if (USE_LAZY_LOADING)
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true);
if (metaFile == null)
{
lazyAddScannedFile(detectedFiles);
}
else
{
immediateAddScannedFile(detectedFiles);
}
}
private void lazyAddScannedFile(Collection<File> detectedFiles)
{
for (File file : detectedFiles)
{
try
{
DhSectionPos pos = decodePositionByFile(file);
if (pos != null)
{
unloadedFiles.put(pos, file);
this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
}
}
catch (Exception e)
{
LOGGER.error("Failed to read data meta file at " + file + ": ", e);
FileUtil.renameCorruptedFile(file);
}
}
}
private void immediateAddScannedFile(Collection<File> detectedFiles)
{
HashMultimap<DhSectionPos, FullDataMetaFile> filesByPos = HashMultimap.create();
{ // Sort files by pos.
for (File file : detectedFiles)
{
try
{
FullDataMetaFile metaFile = new FullDataMetaFile(this, this.level, file);
filesByPos.put(metaFile.pos, metaFile);
}
catch (IOException e)
{
LOGGER.error("Failed to read data meta file at " + file + ": ", e);
FileUtil.renameCorruptedFile(file);
}
}
return CompletableFuture.completedFuture(null);
}
// Warn for multiple files with the same pos, and then select the one with the latest timestamp.
for (DhSectionPos pos : filesByPos.keySet())
{
Collection<FullDataMetaFile> metaFiles = filesByPos.get(pos);
FullDataMetaFile fileToUse;
if (metaFiles.size() > 1)
{
// fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get()));
fileToUse = Collections.max(metaFiles, Comparator.comparingLong(fullDataMetaFile -> fullDataMetaFile.file.lastModified()));
// future wrapper necessary in order to handle file read errors
CompletableFuture<IFullDataSource> futureWrapper = new CompletableFuture<>();
metaFile.loadOrGetCachedDataSourceAsync().exceptionally((e) ->
{
StringBuilder sb = new StringBuilder();
sb.append("Multiple files with the same pos: ");
sb.append(pos);
sb.append("\n");
for (FullDataMetaFile metaFile : metaFiles)
{
sb.append("\t");
sb.append(metaFile.file);
sb.append("\n");
}
sb.append("\tUsing: ");
sb.append(fileToUse.file);
sb.append("\n");
sb.append("(Other files will be renamed by appending \".old\" to their name.)");
LOGGER.warn(sb.toString());
FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, metaFile, e);
// Rename all other files with the same pos to .old
for (FullDataMetaFile metaFile : metaFiles)
{
if (metaFile == fileToUse)
{
continue;
}
File oldFile = new File(metaFile.file + ".old");
try
{
if (!metaFile.file.renameTo(oldFile))
{
throw new RuntimeException("Renaming failed");
}
}
catch (Exception e)
{
LOGGER.error("Failed to rename file: " + metaFile.file + " to " + oldFile, e);
}
}
}
}
else
{
fileToUse = metaFiles.iterator().next();
}
// Add file to the list of files.
this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, fileToUse.pos.sectionDetailLevel));
this.fileBySectionPos.put(pos, fileToUse);
}
futureWrapper.completeExceptionally(e);
return null; // return value doesn't matter
})
.whenComplete((dataSource, e) ->
{
futureWrapper.complete(dataSource);
});
return futureWrapper;
}
@Override
public FullDataMetaFile getFileIfExist(DhSectionPos pos) { return this.getLoadOrMakeFile(pos, false); }
protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile)
{
FullDataMetaFile metaFile = this.fileBySectionPos.get(pos);
if (metaFile != null) return metaFile;
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(pos);
if (metaFile != null)
{
return metaFile;
}
File fileToLoad = this.unloadedFiles.get(pos);
File fileToLoad = this.unloadedFileBySectionPos.get(pos);
// File does exist, but not loaded yet.
if (fileToLoad != null)
{
@@ -218,7 +165,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
// Double check locking for loading file, as loading file means also loading the metadata, which
// while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the
// duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it.
metaFile = this.fileBySectionPos.get(pos);
metaFile = this.metaFileBySectionPos.get(pos);
if (metaFile != null)
{
return metaFile; // someone else loaded it already.
@@ -227,8 +174,8 @@ public class FullDataFileHandler implements IFullDataSourceProvider
try
{
metaFile = new FullDataMetaFile(this, this.level, fileToLoad);
this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
this.fileBySectionPos.put(pos, metaFile);
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
this.metaFileBySectionPos.put(pos, metaFile);
return metaFile;
}
catch (IOException e)
@@ -238,7 +185,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
}
finally
{
this.unloadedFiles.remove(pos);
this.unloadedFileBySectionPos.remove(pos);
}
}
}
@@ -262,10 +209,10 @@ public class FullDataFileHandler implements IFullDataSourceProvider
return null;
}
this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
// This is a CAS with expected null value.
FullDataMetaFile metaFileCas = this.fileBySectionPos.putIfAbsent(pos, metaFile);
FullDataMetaFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile);
return metaFileCas == null ? metaFile : metaFileCas;
}
@@ -304,7 +251,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
}
// check if a file for this pos exists, either loaded and unloaded
if (this.fileBySectionPos.containsKey(subPos) || this.unloadedFiles.containsKey(subPos))
if (this.metaFileBySectionPos.containsKey(subPos) || this.unloadedFileBySectionPos.containsKey(subPos))
{
allEmpty = false;
break outerLoop;
@@ -334,13 +281,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider
if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos))
{
// load the file if it isn't already
if (this.unloadedFiles.containsKey(childPos))
if (this.unloadedFileBySectionPos.containsKey(childPos))
{
this.getLoadOrMakeFile(childPos, true);
}
FullDataMetaFile metaFile = this.fileBySectionPos.get(childPos);
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(childPos);
if (metaFile != null)
{
// we have reached a populated leaf node in the quad tree
@@ -359,42 +306,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider
}
}
public void ForEachFile(Consumer<FullDataMetaFile> consumer) { this.metaFileBySectionPos.values().forEach(consumer); }
/**
* Returns the {@link IFullDataSource} for the given section position. <Br>
* The returned data source may be null. <Br> <Br>
*
* For now, if result is null, it prob means error has occurred when loading or creating the file object. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
@Override
public CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos)
{
this.topDetailLevel.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true);
if (metaFile == null)
{
return CompletableFuture.completedFuture(null);
}
// future wrapper necessary in order to handle file read errors
CompletableFuture<IFullDataSource> futureWrapper = new CompletableFuture<>();
metaFile.loadOrGetCachedDataSourceAsync().exceptionally((e) ->
{
FullDataMetaFile newMetaFile = this.removeCorruptedFile(pos, metaFile, e);
futureWrapper.completeExceptionally(e);
return null; // return value doesn't matter
})
.whenComplete((dataSource, e) ->
{
futureWrapper.complete(dataSource);
});
return futureWrapper;
}
//=============//
// data saving //
//=============//
/** This call is concurrent. I.e. it supports being called by multiple threads at the same time. */
@Override
@@ -408,14 +326,14 @@ public class FullDataFileHandler implements IFullDataSourceProvider
}
private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData)
{
FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos);
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos);
if (metaFile != null)
{
// there is a file for this position
metaFile.addToWriteQueue(chunkData);
}
if (sectionPos.sectionDetailLevel <= this.topDetailLevel.get())
if (sectionPos.sectionDetailLevel <= this.topDetailLevelRef.get())
{
// recursively attempt to get the meta file for this position
this.writeChunkDataToMetaFile(sectionPos.getParentPos(), chunkData);
@@ -427,7 +345,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
public CompletableFuture<Void> flushAndSave()
{
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
for (FullDataMetaFile metaFile : this.fileBySectionPos.values())
for (FullDataMetaFile metaFile : this.metaFileBySectionPos.values())
{
futures.add(metaFile.flushAndSaveAsync());
}
@@ -437,7 +355,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
@Override
public CompletableFuture<Void> flushAndSave(DhSectionPos sectionPos)
{
FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos);
FullDataMetaFile metaFile = this.metaFileBySectionPos.get(sectionPos);
if (metaFile == null)
{
return CompletableFuture.completedFuture(null);
@@ -446,11 +364,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
}
@Override
public synchronized void addOnUpdatedListener(Consumer<IFullDataSource> listener)
{
this.onUpdatedListeners.add(listener);
}
protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos)
{
@@ -526,7 +440,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider
FileUtil.renameCorruptedFile(metaFile.file);
// remove the FullDataMetaFile since the old one was corrupted
this.fileBySectionPos.remove(pos);
this.metaFileBySectionPos.remove(pos);
// create a new FullDataMetaFile to write new data to
return this.getLoadOrMakeFile(pos, true);
}
@@ -537,10 +451,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider
Consumer<IFullDataSource> onUpdated, Function<IFullDataSource, Boolean> updater)
{
boolean changed = updater.apply(source);
// if (changed)
// {
// metaData.dataVersion.incrementAndGet();
// }
if (source instanceof IIncompleteFullDataSource)
{
@@ -556,17 +466,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider
return CompletableFuture.completedFuture(source);
}
@Override
public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + LOD_FILE_POSTFIX); }
@Nullable
public DhSectionPos decodePositionByFile(File file)
{
String fileName = file.getName();
if (!fileName.endsWith(LOD_FILE_POSTFIX)) return null;
fileName = fileName.substring(0, fileName.length() - 4);
return DhSectionPos.deserialize(fileName);
}
//==========================//
@@ -619,12 +518,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider
@Override
public ExecutorService getIOExecutor() { return fileHandlerThreadPool; }
@Override
public FullDataMetaFile getFileIfExist(DhSectionPos pos)
{
return getLoadOrMakeFile(pos, false);
}
//=========//
@@ -632,9 +525,15 @@ public class FullDataFileHandler implements IFullDataSourceProvider
//=========//
@Override
public void close()
{
FullDataMetaFile.debugPhantomLifeCycleCheck();
}
public void close() { FullDataMetaFile.debugPhantomLifeCycleCheck(); }
//================//
// helper methods //
//================//
@Override
public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + FullDataMetaFile.FILE_SUFFIX); }
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java
+4
View File
@@ -51,8 +51,12 @@ import org.apache.logging.log4j.Logger;
*/
public class FullDataMetaFile extends AbstractMetaDataContainerFile implements IDebugRenderable
{
public static final String FILE_SUFFIX = ".lod";
private static final Logger LOGGER = DhLoggerBuilder.getLogger(FullDataMetaFile.class.getSimpleName());
private final IDhLevel level;
private final IFullDataSourceProvider fullDataSourceProvider;
public boolean doesFileExist;
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java
-12
View File
@@ -57,18 +57,6 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler
//======//
// data //
//======//
@Override
public CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos)
{
return super.readAsync(pos);
}
//==================//
// generation queue //
//==================//
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java
+1 -3
View File
@@ -33,15 +33,13 @@ import java.util.function.Function;
public interface IFullDataSourceProvider extends AutoCloseable
{
void addScannedFile(Collection<File> detectedFiles);
void addScannedFiles(Collection<File> detectedFiles);
CompletableFuture<IFullDataSource> readAsync(DhSectionPos pos);
void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData);
CompletableFuture<Void> flushAndSave();
CompletableFuture<Void> flushAndSave(DhSectionPos sectionPos);
void addOnUpdatedListener(Consumer<IFullDataSource> listener);
//long getCacheVersion(DhSectionPos sectionPos);
//boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion);
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/ILodRenderSourceProvider.java
+3 -4
View File
@@ -35,14 +35,13 @@ import java.util.concurrent.CompletableFuture;
*/
public interface ILodRenderSourceProvider extends AutoCloseable
{
CompletableFuture<ColumnRenderSource> readAsync(DhSectionPos pos);
void addScannedFiles(Collection<File> detectedFiles);
CompletableFuture<ColumnRenderSource> readAsync(DhSectionPos pos);
void writeChunkDataToFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData);
CompletableFuture<Void> flushAndSaveAsync();
/** Returns true if the data was refreshed, false otherwise */
//boolean refreshRenderSource(ColumnRenderSource source);
/** Deletes any data stored in the render cache so it can be re-created */
void deleteRenderCache();
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java
+1
View File
@@ -56,6 +56,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final String FILE_SUFFIX = ".rlod";
public static final boolean ALWAYS_INVALIDATE_CACHE = false;
public static final long RENDER_SOURCE_TYPE_ID = ColumnRenderSource.TYPE_ID;
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java
+22 -136
View File
@@ -19,7 +19,6 @@
package com.seibel.distanthorizons.core.file.renderfile;
import com.google.common.collect.HashMultimap;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -29,12 +28,11 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.util.FileScanUtil;
import com.seibel.distanthorizons.core.util.MetaFileScanUtil;
import com.seibel.distanthorizons.core.util.FileUtil;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
@@ -42,8 +40,6 @@ import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX;
public class RenderSourceFileHandler implements ILodRenderSourceProvider
{
public static final boolean USE_LAZY_LOADING = true;
@@ -60,7 +56,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
private final IDhClientLevel clientLevel;
private final File saveDir;
/** This is the lowest (highest numeric) detail level that this {@link RenderSourceFileHandler} is keeping track of. */
AtomicInteger topDetailLevel = new AtomicInteger(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
AtomicInteger topDetailLevelRef = new AtomicInteger(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
private final IFullDataSourceProvider fullDataSourceProvider;
private final WeakHashMap<CompletableFuture<?>, ETaskType> taskTracker = new WeakHashMap<>();
@@ -85,136 +81,35 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log);
FileScanUtil.scanRenderFiles(saveStructure, clientLevel.getLevelWrapper(), this);
MetaFileScanUtil.scanRenderFiles(saveStructure, clientLevel.getLevelWrapper(), this);
}
/**
* Caller must ensure that this method is called only once,
* and that the given files are not used before this method is called. <br><br>
*
* Used by {@link FileScanUtil#scanRenderFiles(AbstractSaveStructure, ILevelWrapper, ILodRenderSourceProvider)}
* Used by {@link MetaFileScanUtil#scanRenderFiles(AbstractSaveStructure, ILevelWrapper, ILodRenderSourceProvider)}
*/
@Override
public void addScannedFiles(Collection<File> detectedFiles)
{
if (USE_LAZY_LOADING)
{
this.lazyAddScannedFile(detectedFiles);
}
else
{
this.immediateAddScannedFile(detectedFiles);
}
}
private void lazyAddScannedFile(Collection<File> detectedFiles)
{
for (File file : detectedFiles)
{
if (file == null || !file.exists())
{
// can rarely happen if the user rapidly travels between dimensions
LOGGER.warn("Null or non-existent render file: " + ((file != null) ? file.getPath() : "NULL"));
continue;
}
try
{
DhSectionPos pos = this.decodePositionFromFileName(file);
if (pos != null)
{
this.unloadedFileBySectionPos.put(pos, file);
this.topDetailLevel.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.sectionDetailLevel));
}
}
catch (Exception e)
{
LOGGER.error("Failed to read data meta file at " + file + ": ", e);
FileUtil.renameCorruptedFile(file);
}
}
}
private void immediateAddScannedFile(Collection<File> newRenderFiles)
{
HashMultimap<DhSectionPos, RenderMetaDataFile> filesByPos = HashMultimap.create();
MetaFileScanUtil.CreateMetadataFunc createMetadataFunc = (file) -> RenderMetaDataFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, file);
// Sort files by pos.
for (File file : newRenderFiles)
MetaFileScanUtil.AddUnloadedFileFunc addUnloadedFileFunc = (pos, file) ->
{
try
{
RenderMetaDataFile metaFile = RenderMetaDataFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, file);
filesByPos.put(metaFile.pos, metaFile);
}
catch (IOException e)
{
LOGGER.error("Failed to read render meta file at [" + file + "]. Error: ", e);
FileUtil.renameCorruptedFile(file);
}
}
this.unloadedFileBySectionPos.put(pos, file);
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
};
MetaFileScanUtil.AddLoadedMetaFileFunc addLoadedMetaFileFunc = (pos, loadedMetaFile) ->
{
this.topDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.sectionDetailLevel));
this.metaFileBySectionPos.put(pos, (RenderMetaDataFile) loadedMetaFile);
};
// Warn for multiple files with the same pos, and then select the one with the latest timestamp.
for (DhSectionPos pos : filesByPos.keySet())
{
Collection<RenderMetaDataFile> metaFiles = filesByPos.get(pos);
RenderMetaDataFile fileToUse;
if (metaFiles.size() > 1)
{
//fileToUse = metaFiles.stream().findFirst().orElse(null); // use the first file in the list
// use the file's last modified date
fileToUse = Collections.max(metaFiles, Comparator.comparingLong(renderMetaDataFile ->
renderMetaDataFile.file.lastModified()));
// fileToUse = Collections.max(metaFiles, Comparator.comparingLong(renderMetaDataFile ->
// renderMetaDataFile.metaData.dataVersion.get()));
{
StringBuilder sb = new StringBuilder();
sb.append("Multiple files with the same pos: ");
sb.append(pos);
sb.append("\n");
for (RenderMetaDataFile metaFile : metaFiles)
{
sb.append("\t");
sb.append(metaFile.file);
sb.append("\n");
}
sb.append("\tUsing: ");
sb.append(fileToUse.file);
sb.append("\n");
sb.append("(Other files will be renamed by appending \".old\" to their name.)");
LOGGER.warn(sb.toString());
// Rename all other files with the same pos to .old
for (RenderMetaDataFile metaFile : metaFiles)
{
if (metaFile == fileToUse)
{
continue;
}
File oldFile = new File(metaFile.file + ".old");
try
{
if (!metaFile.file.renameTo(oldFile))
throw new RuntimeException("Renaming failed");
}
catch (Exception e)
{
LOGGER.error("Failed to rename file: [" + metaFile.file + "] to [" + oldFile + "]", e);
}
}
}
}
else
{
fileToUse = metaFiles.iterator().next();
}
// Add this file to the list of files.
this.metaFileBySectionPos.put(pos, fileToUse);
// increase the lowest detail level if a new lower detail file is found
this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
}
MetaFileScanUtil.addScannedFiles(detectedFiles, USE_LAZY_LOADING, RenderMetaDataFile.FILE_SUFFIX,
createMetadataFunc,
addUnloadedFileFunc, addLoadedMetaFileFunc);
}
@@ -300,7 +195,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
try
{
metaFile = RenderMetaDataFile.createFromExistingFile(this.fullDataSourceProvider, this.clientLevel, fileToLoad);
this.topDetailLevel.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.sectionDetailLevel));
this.topDetailLevelRef.updateAndGet(currentTopDetailLevel -> Math.max(currentTopDetailLevel, pos.sectionDetailLevel));
this.metaFileBySectionPos.put(pos, metaFile);
return metaFile;
}
@@ -326,7 +221,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
// due to a rare issue where the file may already exist but isn't in the file list
metaFile = RenderMetaDataFile.createFromExistingOrNewFile(this.clientLevel, this.fullDataSourceProvider, pos, this.computeRenderFilePath(pos));
this.topDetailLevel.updateAndGet(newDetailLevel -> Math.max(newDetailLevel, pos.sectionDetailLevel));
this.topDetailLevelRef.updateAndGet(newDetailLevel -> Math.max(newDetailLevel, pos.sectionDetailLevel));
// Compare And Swap to handle a concurrency issue where multiple threads created the same Meta File at the same time
RenderMetaDataFile metaFileCas = this.metaFileBySectionPos.putIfAbsent(pos, metaFile);
@@ -375,7 +270,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
}
}
if (sectionDetailLevel < this.topDetailLevel.get())
if (sectionDetailLevel < this.topDetailLevelRef.get())
{
this.writeChunkDataToFileRecursively(chunk, (byte) (sectionDetailLevel + 1));
}
@@ -483,16 +378,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
// helper methods //
//================//
public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RENDER_FILE_POSTFIX); }
@Nullable
public DhSectionPos decodePositionFromFileName(File file)
{
String fileName = file.getName();
if (!fileName.endsWith(RENDER_FILE_POSTFIX)) return null;
fileName = fileName.substring(0, fileName.length() - RENDER_FILE_POSTFIX.length());
return DhSectionPos.deserialize(fileName);
}
public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RenderMetaDataFile.FILE_SUFFIX); }
core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java
-78
View File
@@ -1,78 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.util;
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/** Used to pull in the initial files used by both {@link IFullDataSourceProvider} and {@link ILodRenderSourceProvider}s. */
public class FileScanUtil
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final int MAX_SCAN_DEPTH = 5;
public static final String LOD_FILE_POSTFIX = ".lod";
public static final String RENDER_FILE_POSTFIX = ".rlod";
public static void scanFullDataFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, IFullDataSourceProvider dataSourceProvider)
{
try (Stream<Path> pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
{
List<File> files = pathStream.filter(
path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile()
).map(Path::toFile).collect(Collectors.toList());
LOGGER.info("Found " + files.size() + " full data files for " + levelWrapper + " in " + saveStructure);
dataSourceProvider.addScannedFile(files);
}
catch (Exception e)
{
LOGGER.error("Failed to scan and collect full data files for " + levelWrapper + " in " + saveStructure, e);
}
}
public static void scanRenderFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, ILodRenderSourceProvider renderSourceProvider)
{
try (Stream<Path> pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
{
List<File> files = pathStream.filter(
path -> path.toFile().getName().endsWith(RENDER_FILE_POSTFIX) && path.toFile().isFile()
).map(Path::toFile).collect(Collectors.toList());
LOGGER.info("Found " + files.size() + " render cache files for " + levelWrapper + " in " + saveStructure);
renderSourceProvider.addScannedFiles(files);
}
catch (Exception e)
{
LOGGER.error("Failed to scan and collect cache files for " + levelWrapper + " in " + saveStructure, e);
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/util/MetaFileScanUtil.java
+257
View File
@@ -0,0 +1,257 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.util;
import com.google.common.collect.HashMultimap;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandler;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.metaData.AbstractMetaDataContainerFile;
import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider;
import com.seibel.distanthorizons.core.file.renderfile.RenderMetaDataFile;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/** Used to pull in the initial files used by both {@link IFullDataSourceProvider} and {@link ILodRenderSourceProvider}s. */
public class MetaFileScanUtil
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final int MAX_SCAN_DEPTH = 5;
//===============//
// file scanning //
//===============//
// file scanning means to find all File's in a given directory
// TODO merge with the below method
public static void scanFullDataFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, IFullDataSourceProvider dataSourceProvider)
{
try (Stream<Path> pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
{
List<File> files = pathStream.filter(
path -> path.toFile().getName().endsWith(FullDataMetaFile.FILE_SUFFIX) && path.toFile().isFile()
).map(Path::toFile).collect(Collectors.toList());
LOGGER.info("Found " + files.size() + " full data files for " + levelWrapper + " in " + saveStructure);
dataSourceProvider.addScannedFiles(files);
}
catch (Exception e)
{
LOGGER.error("Failed to scan and collect full data files for " + levelWrapper + " in " + saveStructure, e);
}
}
// TODO merge with the above method
public static void scanRenderFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, ILodRenderSourceProvider renderSourceProvider)
{
try (Stream<Path> pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH))
{
List<File> files = pathStream.filter(
path -> path.toFile().getName().endsWith(RenderMetaDataFile.FILE_SUFFIX) && path.toFile().isFile()
).map(Path::toFile).collect(Collectors.toList());
LOGGER.info("Found " + files.size() + " render cache files for " + levelWrapper + " in " + saveStructure);
renderSourceProvider.addScannedFiles(files);
}
catch (Exception e)
{
LOGGER.error("Failed to scan and collect cache files for " + levelWrapper + " in " + saveStructure, e);
}
}
//======================//
// Adding scanned files //
//======================//
/**
* Caller must ensure that this method is called only once,
* and that the {@link FullDataFileHandler} is not used before this method is called.
*/
public static void addScannedFiles(
Collection<File> detectedFiles, boolean useLazyLoading, String fileSuffix,
CreateMetadataFunc createMetadataFunc,
AddUnloadedFileFunc addUnloadedFileFunc, AddLoadedMetaFileFunc addLoadedMetaFileFunc)
{
if (useLazyLoading)
{
lazyAddScannedFile(detectedFiles, fileSuffix, addUnloadedFileFunc);
}
else
{
immediateAddScannedFile(detectedFiles, createMetadataFunc, addLoadedMetaFileFunc);
}
}
private static void lazyAddScannedFile(Collection<File> detectedFiles, String fileSuffix, AddUnloadedFileFunc addUnloadedFileFunc)
{
for (File file : detectedFiles)
{
if (file == null || !file.exists())
{
// can rarely happen if the user rapidly travels between dimensions
LOGGER.warn("Null or non-existent file: " + ((file != null) ? file.getPath() : "NULL"));
continue;
}
try
{
DhSectionPos pos = decodePositionFromFileName(file, fileSuffix);
if (pos != null)
{
addUnloadedFileFunc.addFile(pos, file);
}
}
catch (Exception e)
{
LOGGER.error("Failed to read data meta file at " + file + ": ", e);
FileUtil.renameCorruptedFile(file);
}
}
}
private static void immediateAddScannedFile(
Collection<File> detectedFiles,
CreateMetadataFunc createMetadataFunc, AddLoadedMetaFileFunc addLoadedMetaFileFunc)
{
HashMultimap<DhSectionPos, AbstractMetaDataContainerFile> filesByPos = HashMultimap.create();
{ // Sort files by pos.
for (File file : detectedFiles)
{
try
{
AbstractMetaDataContainerFile metaFile = createMetadataFunc.createFile(file);
filesByPos.put(metaFile.pos, metaFile);
}
catch (IOException e)
{
LOGGER.error("Failed to read data meta file at " + file + ": ", e);
FileUtil.renameCorruptedFile(file);
}
}
}
// Warn for multiple files with the same pos, and then select the one with the latest timestamp.
for (DhSectionPos pos : filesByPos.keySet())
{
Collection<AbstractMetaDataContainerFile> metaFiles = filesByPos.get(pos);
AbstractMetaDataContainerFile metaFileToUse;
if (metaFiles.size() > 1)
{
// sort by the file's last modified date
metaFileToUse = Collections.max(metaFiles, Comparator.comparingLong(fullDataMetaFile -> fullDataMetaFile.file.lastModified()));
// log the duplicate files
StringBuilder duplicateMessage = new StringBuilder();
duplicateMessage.append("Multiple files with the same pos: ").append(pos).append("\n");
for (AbstractMetaDataContainerFile metaFile : metaFiles)
{
duplicateMessage.append("\t").append(metaFile.file).append("\n");
}
duplicateMessage.append("\tUsing: ").append(metaFileToUse.file).append("\n");
duplicateMessage.append("(Other files will be renamed by appending \".old\" to their name.)");
LOGGER.warn(duplicateMessage.toString());
// Rename all other files with the same pos to .old
for (AbstractMetaDataContainerFile metaFile : metaFiles)
{
if (metaFile == metaFileToUse)
{
continue;
}
File oldFile = new File(metaFile.file + ".old");
try
{
if (!metaFile.file.renameTo(oldFile))
{
throw new RuntimeException("Renaming failed");
}
}
catch (Exception e)
{
LOGGER.error("Failed to rename file: " + metaFile.file + " to " + oldFile, e);
}
}
}
else
{
metaFileToUse = metaFiles.iterator().next();
}
// Add file to the list of files.
addLoadedMetaFileFunc.addFile(pos, metaFileToUse);
}
}
//================//
// helper methods //
//================//
/** @return null if the file name can't be parsed into a {@link DhSectionPos} */
@Nullable
public static DhSectionPos decodePositionFromFileName(File file, String fileSuffix)
{
String fileName = file.getName();
if (!fileName.endsWith(fileSuffix))
{
return null;
}
fileName = fileName.substring(0, fileName.length() - 4);
return DhSectionPos.deserialize(fileName);
}
//===================//
// helper interfaces //
//===================//
@FunctionalInterface
public interface CreateMetadataFunc { AbstractMetaDataContainerFile createFile(File file) throws IOException; }
@FunctionalInterface
public interface AddUnloadedFileFunc { void addFile(DhSectionPos pos, File file); }
@FunctionalInterface
public interface AddLoadedMetaFileFunc { void addFile(DhSectionPos pos, AbstractMetaDataContainerFile metaFile); }
}