wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve corrupted Data File handling

Browse Source
This commit is contained in:
James Seibel
2023-02-13 20:03:54 -06:00
parent 5116cef99a
commit 228ba2b2e2
5 changed files with 120 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/lod/core/datatype/transform/DataRenderTransformer.java
+5 -1
View File
@@ -28,7 +28,11 @@ public class DataRenderTransformer
private static ILodRenderSource transform(ILodDataSource dataSource, IDhClientLevel level)
{
if (dataSource == null) return null;
if (dataSource == null)
{
return null;
}
return ColumnRenderLoader.LOADER_BY_SOURCE_TYPE.get(ColumnRenderSource.class)
.stream().findFirst().get().createRenderSource(dataSource, level);
}
core/src/main/java/com/seibel/lod/core/file/FileUtil.java
+50
View File
@@ -0,0 +1,50 @@
package com.seibel.lod.core.file;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import org.apache.logging.log4j.Logger;
import java.io.File;
public class FileUtil
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
/**
* Renames the given file to FILE_NAME.ORIGINAL_PREFIX.corrupted.
* If an existing corrupted file already exists, this will attempt to remove it first.
*
* @return the file after it has been renamed
*/
public static File renameCorruptedFile(File file)
{
String corruptedFileName = file.getName() + ".corrupted";
File corruptedFile = new File(file.getParentFile(), corruptedFileName);
if (corruptedFile.exists())
{
// could happen if there was a corrupted file before that was removed
if (!corruptedFile.delete())
{
LOGGER.error("Unable to delete pre-existing corrupted file ["+corruptedFileName+"].");
}
}
if (file.renameTo(corruptedFile))
{
LOGGER.error("Renamed corrupted file to ["+corruptedFileName+"].");
}
else
{
LOGGER.error("Failed to rename corrupted file to ["+corruptedFileName+"]. Attempting to delete file...");
if (!file.delete())
{
LOGGER.error("Unable to delete corrupted file ["+corruptedFileName+"].");
}
}
return corruptedFile;
}
}
core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java
+38 -14
View File
@@ -264,13 +264,29 @@ public class DataFileHandler implements IDataSourceProvider
@Override
public CompletableFuture<ILodDataSource> read(DhSectionPos pos)
{
this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel));
this.topDetailLevel.updateAndGet(intVal -> Math.max(intVal, pos.sectionDetailLevel));
DataMetaFile metaFile = this.getOrMakeFile(pos);
if (metaFile == null)
{
return CompletableFuture.completedFuture(null);
}
return metaFile.loadOrGetCached();
// future wrapper necessary in order to handle file read errors
CompletableFuture<ILodDataSource> futureWrapper = new CompletableFuture<>();
metaFile.loadOrGetCachedAsync().exceptionally((e) ->
{
DataMetaFile newMetaFile = this.removeCorruptedFile(pos, metaFile, e);
futureWrapper.completeExceptionally(e);
return null; // return value doesn't matter
})
.whenComplete((dataSource, e) ->
{
futureWrapper.complete(dataSource);
});
return futureWrapper;
}
/** This call is concurrent. I.e. it supports being called by multiple threads at the same time. */
@@ -363,8 +379,7 @@ public class DataFileHandler implements IDataSourceProvider
for (DataMetaFile metaFile : existFiles)
{
futures.add(f.loadOrGetCached()
.exceptionally((ex) -> null)
futures.add(metaFile.loadOrGetCachedAsync()
.thenAccept((data) ->
{
if (data != null)
@@ -373,14 +388,30 @@ public class DataFileHandler implements IDataSourceProvider
dataSource.sampleFrom(data);
}
})
.exceptionally((e) ->
{
DataMetaFile newMetaFile = this.removeCorruptedFile(pos, metaFile, e);
return null;
})
);
}
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new))
.thenApply((v) -> dataSource.trySelfPromote());
}
}
private DataMetaFile removeCorruptedFile(DhSectionPos pos, DataMetaFile metaFile, Throwable exception)
{
LOGGER.error("Error reading Data file ["+pos+"]", exception);
FileUtil.renameCorruptedFile(metaFile.path);
// remove the DataMetaFile since the old one was corrupted
this.files.remove(pos);
// create a new DataMetaFile to write new data to
return this.getOrMakeFile(pos);
}
@Override
@Override
public ILodDataSource onDataFileLoaded(ILodDataSource source, MetaData metaData,
Consumer<ILodDataSource> onUpdated, Function<ILodDataSource, Boolean> updater)
{
@@ -426,17 +457,10 @@ public class DataFileHandler implements IDataSourceProvider
}
@Override
public File computeDataFilePath(DhSectionPos pos)
{
return new File(this.saveDir, pos.serialize() + ".lod");
}
public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + ".lod"); }
@Override
public Executor getIOExecutor()
{
return this.fileReaderThread;
}
public Executor getIOExecutor() { return this.fileReaderThread; }
@Override
public void close()
core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java
+24 -18
View File
@@ -151,22 +151,22 @@ public class DataMetaFile extends AbstractMetaDataFile
{
debugCheck();
Object obj = this.data.get();
CompletableFuture<ILodDataSource> cached = this._readCachedAsync(obj);
if (cached != null)
{
return cached;
}
CompletableFuture<ILodDataSource> future = new CompletableFuture<>();
// Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :(
boolean worked = this.data.compareAndSet(obj, future);
boolean worked = this.data.compareAndSet(obj, future); // TODO obj and future are different object types, would this ever return true?
if (!worked)
{
return this.loadOrGetCachedAsync();
}
// After cas. We are in exclusive control.
if (!this.doesFileExist)
{
@@ -181,6 +181,7 @@ public class DataMetaFile extends AbstractMetaDataFile
{
if (e != null)
{
LOGGER.error("Uncaught error on creation {}: ", this.path, e);
future.complete(null);
this.data.set(null);
}
@@ -198,7 +199,7 @@ public class DataMetaFile extends AbstractMetaDataFile
{
if (this.metaData == null)
{
throw new IllegalStateException("Meta data not loaded!");
throw new IllegalStateException("Meta data not loaded!"); // TODO should this be a CompletionException?
}
// Load the file.
@@ -207,10 +208,12 @@ public class DataMetaFile extends AbstractMetaDataFile
{
data = this.loader.loadData(this, fio, this.level);
}
catch (IOException e)
catch (Exception e)
{
// can happen if there is a missing file or the file was incorrectly formatted
throw new CompletionException(e);
}
// Apply the write queue
LodUtil.assertTrue(this.inCacheWriteAccessFuture.get() == null,
"No one should be writing to the cache while we are in the process of " +
@@ -218,17 +221,20 @@ public class DataMetaFile extends AbstractMetaDataFile
data = this.handler.onDataFileLoaded(data, this.metaData, this::saveChanges, this::applyWriteQueue);
return data;
}, handler.getIOExecutor())
.whenComplete((f, e) -> {
if (e != null) {
LOGGER.error("Error loading file {}: ", path, e);
future.complete(null);
data.set(null);
} else {
future.complete(f);
new DataObjTracker(f);
this.data.set(new SoftReference<>(f));
}
}, this.handler.getIOExecutor())
.exceptionally((e) ->
{
LOGGER.error("Error loading file {}: ", this.path, e);
this.data.set(null);
future.completeExceptionally(e);
return null; // the return value here doesn't matter
})
.whenComplete((dataSource, e) ->
{
future.complete(dataSource);
new DataObjTracker(dataSource);
this.data.set(new SoftReference<>(dataSource));
});
}
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java
+3 -7
View File
@@ -290,14 +290,10 @@ public class RenderFileHandler implements ILodRenderSourceProvider
return dataSource;
}).exceptionally((ex) ->
{
if (ex != null)
{
LOGGER.error("Uncaught exception when getting data for updateCache()", ex);
}
LOGGER.error("Exception when getting data for updateCache()", ex);
return null;
});
LOGGER.info("Recreating cache for {}", data.getSectionPos());
DataRenderTransformer.asyncTransformDataSource(dataFuture, this.level)
.thenAccept((newRenderDataSource) -> this.write(dataRef.get(), file, newRenderDataSource, this.dataSourceProvider.getCacheVersion(data.getSectionPos())))
@@ -310,7 +306,7 @@ public class RenderFileHandler implements ILodRenderSourceProvider
return null;
}).thenRun(() -> this.cacheUpdateLockBySectionPos.remove(file.pos));
}
public ILodRenderSource onRenderFileLoaded(ILodRenderSource data, RenderMetaDataFile file)
{
if (!this.dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get()))