wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor MetaData and MetaDataFile

Browse Source
This commit is contained in:
James Seibel
2022-12-25 21:42:00 -06:00
parent b96622f1cd
commit e26ac63dff
15 changed files with 888 additions and 627 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/lod/core/datatype/AbstractRenderSourceLoader.java
+2 -2
View File
@@ -3,7 +3,7 @@ package com.seibel.lod.core.datatype;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.file.renderfile.RenderMetaFile;
import com.seibel.lod.core.file.renderfile.RenderMetaDataFile;
import java.io.IOException;
import java.io.InputStream;
@@ -64,7 +64,7 @@ public abstract class AbstractRenderSourceLoader
}
/** Can return null if the file is out of date or something */
public abstract ILodRenderSource loadRender(RenderMetaFile renderFile, InputStream data, IDhLevel level) throws IOException;
public abstract ILodRenderSource loadRender(RenderMetaDataFile renderFile, InputStream data, IDhLevel level) throws IOException;
public abstract ILodRenderSource createRender(ILodDataSource dataSource, IDhClientLevel level);
core/src/main/java/com/seibel/lod/core/datatype/ILodRenderSource.java
+2 -2
View File
@@ -5,7 +5,7 @@ import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.render.LodQuadTree;
import com.seibel.lod.core.render.RenderBuffer;
import com.seibel.lod.core.file.renderfile.RenderMetaFile;
import com.seibel.lod.core.file.renderfile.RenderMetaDataFile;
import java.io.IOException;
import java.io.OutputStream;
@@ -32,7 +32,7 @@ public interface ILodRenderSource
*/
boolean trySwapRenderBuffer(LodQuadTree quadTree, AtomicReference<RenderBuffer> referenceSlot);
void saveRender(IDhClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException;
void saveRender(IDhClientLevel level, RenderMetaDataFile file, OutputStream dataStream) throws IOException;
byte getRenderVersion();
core/src/main/java/com/seibel/lod/core/datatype/PlaceHolderRenderSource.java
+2 -2
View File
@@ -5,7 +5,7 @@ import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.render.LodQuadTree;
import com.seibel.lod.core.render.RenderBuffer;
import com.seibel.lod.core.file.renderfile.RenderMetaFile;
import com.seibel.lod.core.file.renderfile.RenderMetaDataFile;
import java.io.IOException;
import java.io.OutputStream;
@@ -37,7 +37,7 @@ public class PlaceHolderRenderSource implements ILodRenderSource
public boolean trySwapRenderBuffer(LodQuadTree quadTree, AtomicReference<RenderBuffer> referenceSlots) { return false; }
@Override
public void saveRender(IDhClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException
public void saveRender(IDhClientLevel level, RenderMetaDataFile file, OutputStream dataStream) throws IOException
{
throw new UnsupportedOperationException("EmptyRenderSource should NEVER be saved!");
}
core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java
+2 -2
View File
@@ -8,7 +8,7 @@ import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.datatype.ILodRenderSource;
import com.seibel.lod.core.datatype.AbstractRenderSourceLoader;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.file.renderfile.RenderMetaFile;
import com.seibel.lod.core.file.renderfile.RenderMetaDataFile;
import com.seibel.lod.core.util.LodUtil;
import java.io.DataInputStream;
@@ -22,7 +22,7 @@ public class ColumnRenderLoader extends AbstractRenderSourceLoader
}
@Override
public ILodRenderSource loadRender(RenderMetaFile dataFile, InputStream data, IDhLevel level) throws IOException {
public ILodRenderSource loadRender(RenderMetaDataFile dataFile, InputStream data, IDhLevel level) throws IOException {
DataInputStream dis = new DataInputStream(data); // DO NOT CLOSE
return new ColumnRenderSource(dataFile.pos, dis, dataFile.metaData.loaderVersion, level);
}
core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java
+2 -2
View File
@@ -7,7 +7,7 @@ import com.seibel.lod.core.datatype.transform.FullToColumnTransformer;
import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.render.RenderBuffer;
import com.seibel.lod.core.file.renderfile.RenderMetaFile;
import com.seibel.lod.core.file.renderfile.RenderMetaDataFile;
import com.seibel.lod.core.enums.ELodDirection;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.level.IDhLevel;
@@ -450,7 +450,7 @@ public class ColumnRenderSource implements ILodRenderSource, IColumnDatatype
}
@Override
public void saveRender(IDhClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException
public void saveRender(IDhClientLevel level, RenderMetaDataFile file, OutputStream dataStream) throws IOException
{
DataOutputStream dos = new DataOutputStream(dataStream); // DO NOT CLOSE
writeData(dos);
core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java
+3 -3
View File
@@ -7,7 +7,7 @@ import com.seibel.lod.core.datatype.full.ChunkSizedData;
import com.seibel.lod.core.datatype.full.FullDataSource;
import com.seibel.lod.core.datatype.full.SparseDataSource;
import com.seibel.lod.core.datatype.full.SpottyDataSource;
import com.seibel.lod.core.file.subDimMatching.MetaFile;
import com.seibel.lod.core.file.metaData.MetaData;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.pos.DhSectionPos;
@@ -316,7 +316,7 @@ public class DataFileHandler implements IDataSourceProvider {
}
@Override
public ILodDataSource onDataFileLoaded(ILodDataSource source, MetaFile.MetaData metaData,
public ILodDataSource onDataFileLoaded(ILodDataSource source, MetaData metaData,
Consumer<ILodDataSource> onUpdated, Function<ILodDataSource, Boolean> updater) {
boolean changed = updater.apply(source);
if (changed) metaData.dataVersion.incrementAndGet();
@@ -329,7 +329,7 @@ public class DataFileHandler implements IDataSourceProvider {
return source;
}
@Override
public CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated) {
public CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, MetaData metaData, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated) {
return CompletableFuture.supplyAsync(() -> {
ILodDataSource sourceLocal = source;
boolean changed = updater.apply(sourceLocal);
core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java
+3 -2
View File
@@ -11,8 +11,9 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.AbstractDataSourceLoader;
import com.seibel.lod.core.datatype.full.ChunkSizedData;
import com.seibel.lod.core.file.metaData.MetaData;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.file.subDimMatching.MetaFile;
import com.seibel.lod.core.file.metaData.MetaDataFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
@@ -20,7 +21,7 @@ import com.seibel.lod.core.util.AtomicsUtil;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
public class DataMetaFile extends MetaFile
public class DataMetaFile extends MetaDataFile
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger(DataMetaFile.class.getSimpleName());
core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java
+3 -3
View File
@@ -2,7 +2,7 @@ package com.seibel.lod.core.file.datafile;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.full.ChunkSizedData;
import com.seibel.lod.core.file.subDimMatching.MetaFile;
import com.seibel.lod.core.file.metaData.MetaData;
import com.seibel.lod.core.pos.DhSectionPos;
import java.io.File;
@@ -23,8 +23,8 @@ public interface IDataSourceProvider extends AutoCloseable {
boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion);
CompletableFuture<ILodDataSource> onCreateDataFile(DataMetaFile file);
ILodDataSource onDataFileLoaded(ILodDataSource source, MetaFile.MetaData metaData, Consumer<ILodDataSource> onUpdated, Function<ILodDataSource, Boolean> updater);
CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated);
ILodDataSource onDataFileLoaded(ILodDataSource source, MetaData metaData, Consumer<ILodDataSource> onUpdated, Function<ILodDataSource, Boolean> updater);
CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, MetaData metaData, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated);
File computeDataFilePath(DhSectionPos pos);
Executor getIOExecutor();
core/src/main/java/com/seibel/lod/core/file/metaData/MetaData.java
+44
View File
@@ -0,0 +1,44 @@
package com.seibel.lod.core.file.metaData;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import com.seibel.lod.core.util.objects.UnclosableOutputStream;
import org.apache.logging.log4j.Logger;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
/**
* See {@link MetaDataFile} for a byte map inorder to see the currently used bytes
*/
public class MetaData
{
public DhSectionPos pos;
public int checksum;
public AtomicLong dataVersion;
public byte dataLevel;
//Loader stuff
public long dataTypeId;
public byte loaderVersion;
public MetaData(DhSectionPos pos, int checksum, long dataVersion, byte dataLevel, long dataTypeId, byte loaderVersion)
{
this.pos = pos;
this.checksum = checksum;
this.dataVersion = new AtomicLong(dataVersion);
this.dataLevel = dataLevel;
this.dataTypeId = dataTypeId;
this.loaderVersion = loaderVersion;
}
}
core/src/main/java/com/seibel/lod/core/file/metaData/MetaDataFile.java
+245
View File
@@ -0,0 +1,245 @@
package com.seibel.lod.core.file.metaData;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.objects.UnclosableOutputStream;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
/**
* Used size: 40 bytes <br>
* Remaining space: 24 bytes <br>
* Total size: 64 bytes <br> <br><br>
*
*
* Metadata format: <br> <br>
*
* 4 bytes: metadata identifier bytes: "DHv0" (in ascii: 0x44 48 76 30) this signals the file is in the metadata format <br>
* 4 bytes: section X position <br>
* 4 bytes: section Y position (Unused, for future proofing) <br>
* 4 bytes: section Z position <br> <br>
*
* 4 bytes: data checksum <br> //TODO: Implement checksum
* 1 byte: section detail level <br>
* 1 byte: data detail level // Note: not sure if this is needed <br>
* 1 byte: loader version <br>
* 1 byte: unused <br> <br>
*
* 8 bytes: datatype identifier <br> <br>
*
* 8 bytes: data version
*/
public abstract class MetaDataFile
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final int METADATA_SIZE = 64;
public static final int METADATA_RESERVED_SIZE = 24;
/** equivalent to "DHv0" */
public static final int METADATA_IDENTITY_BYTES = 0x44_48_76_30;
/**
* Currently set to false because for some reason
* Window is throwing PermissionDeniedException when trying to atomic replace a file...
*/
public static final boolean USE_ATOMIC_MOVE_REPLACE = false;
public volatile MetaData metaData = null;
/** also defined in {@link MetaDataFile#metaData} */
public final DhSectionPos pos;
public File path;
//==============//
// constructors //
//==============//
/** Create a metaFile in this path. If the path has a file, throws FileAlreadyExistsException */
protected MetaDataFile(File path, DhSectionPos pos) throws IOException
{
this.path = path;
this.pos = pos;
if (path.exists())
{
throw new FileAlreadyExistsException(path.toString());
}
}
/**
* Creates a {@link MetaDataFile} with the file at the given path.
* @throws IOException if the file was formatted incorrectly
* @throws FileNotFoundException if no file exists for the given path
*/
protected MetaDataFile(File path) throws IOException, FileNotFoundException
{
this.path = path;
if (!path.exists())
{
throw new FileNotFoundException("File not found at [" + path + "]");
}
validateMetaDataFile(this.path);
this.metaData = readMetaDataFromFile(path);
this.pos = this.metaData.pos;
}
/**
* Attempts to create a new {@link MetaDataFile} from the given file.
* @throws IOException if the file was formatted incorrectly
*/
private static MetaData readMetaDataFromFile(File file) throws IOException
{
try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ))
{
ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE);
channel.read(buffer, 0);
channel.close();
buffer.flip();
int idBytes = buffer.getInt();
if (idBytes != METADATA_IDENTITY_BYTES)
{
throw new IOException("Invalid file format: Metadata Identity byte check failed. Expected: [" + METADATA_IDENTITY_BYTES + "], Actual: [" + idBytes + "].");
}
int x = buffer.getInt();
int y = buffer.getInt(); // Unused
int z = buffer.getInt();
int checksum = buffer.getInt();
byte detailLevel = buffer.get();
byte dataLevel = buffer.get();
byte loaderVersion = buffer.get();
byte unused = buffer.get();
long dataTypeId = buffer.getLong();
long timestamp = buffer.getLong();
LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE);
DhSectionPos dataPos = new DhSectionPos(detailLevel, x, z);
return new MetaData(dataPos, checksum, timestamp, dataLevel, dataTypeId, loaderVersion);
}
}
//================//
// helper methods //
//================//
/** Throws an {@link IOException} if the given file isn't valid */
private static void validateMetaDataFile(File file) throws IOException
{
if (!file.exists()) throw new IOException("File missing");
if (!file.isFile()) throw new IOException("Not a file");
if (!file.canRead()) throw new IOException("File not readable");
if (!file.canWrite()) throw new IOException("File not writable");
}
/** Sets this object's {@link MetaDataFile#metaData} using the set {@link MetaDataFile#path} */
protected void loadMetaData() throws IOException
{
validateMetaDataFile(this.path);
this.metaData = readMetaDataFromFile(this.path);
if (!this.metaData.pos.equals(this.pos))
{
LOGGER.warn("The file is from a different location than expected! Expected: [{}] but got [{}]. Ignoring file tag.", this.pos, this.metaData.pos);
this.metaData.pos = this.pos;
}
}
protected void writeData(IMetaDataWriter<OutputStream> dataWriter) throws IOException
{
LodUtil.assertTrue(this.metaData != null);
if (this.path.exists())
{
validateMetaDataFile(this.path);
}
File writerFile;
if (USE_ATOMIC_MOVE_REPLACE)
{
writerFile = new File(this.path.getPath() + ".tmp");
writerFile.deleteOnExit();
}
else
{
writerFile = this.path;
}
try (FileChannel file = FileChannel.open(writerFile.toPath(),
StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))
{
{
file.position(METADATA_SIZE);
int checksum;
try (OutputStream channelOut = new UnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel
BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer?
CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32()))
{ // TODO: Is Adler32 ok?
dataWriter.writeBufferToFile(checkedOut);
checksum = (int) checkedOut.getChecksum().getValue();
}
file.position(0);
// Write metadata
ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE);
buff.putInt(METADATA_IDENTITY_BYTES);
buff.putInt(this.pos.sectionX);
buff.putInt(Integer.MIN_VALUE); // Unused
buff.putInt(this.pos.sectionZ);
buff.putInt(checksum);
buff.put(this.pos.sectionDetail);
buff.put(this.metaData.dataLevel);
buff.put(this.metaData.loaderVersion);
buff.put(Byte.MIN_VALUE); // Unused
buff.putLong(this.metaData.dataTypeId);
buff.putLong(this.metaData.dataVersion.get());
LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE);
buff.flip();
file.write(buff);
}
file.close();
if (USE_ATOMIC_MOVE_REPLACE)
{
// Atomic move / replace the actual file
Files.move(writerFile.toPath(), this.path.toPath(), StandardCopyOption.ATOMIC_MOVE);
}
}
finally
{
try
{
if (USE_ATOMIC_MOVE_REPLACE && writerFile.exists())
{
boolean fileRemoved = writerFile.delete(); // Delete temp file. Ignore errors if it fails.
}
}
catch (SecurityException ignored) { }
}
}
//================//
// helper classes //
//================//
@FunctionalInterface
public interface IMetaDataWriter<T>
{
void writeBufferToFile(T t) throws IOException;
}
}
core/src/main/java/com/seibel/lod/core/file/renderfile/IRenderSourceProvider.java
+5 -1
View File
@@ -8,10 +8,14 @@ import java.io.File;
import java.util.Collection;
import java.util.concurrent.CompletableFuture;
public interface IRenderSourceProvider extends AutoCloseable {
public interface IRenderSourceProvider extends AutoCloseable
{
CompletableFuture<ILodRenderSource> read(DhSectionPos pos);
void addScannedFile(Collection<File> detectedFiles);
void write(DhSectionPos sectionPos, ChunkSizedData chunkData);
CompletableFuture<Void> flushAndSave();
/** Returns true if the data was refreshed, false otherwise */
boolean refreshRenderSource(ILodRenderSource source);
}
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java
+286 -193
View File
@@ -26,220 +26,305 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
public class RenderFileHandler implements IRenderSourceProvider {
public class RenderFileHandler implements IRenderSourceProvider
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
final ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread");
final ConcurrentHashMap<DhSectionPos, RenderMetaFile> files = new ConcurrentHashMap<>();
final IDhClientLevel level;
final File saveDir;
final IDataSourceProvider dataSourceProvider;
public RenderFileHandler(IDataSourceProvider sourceProvider, IDhClientLevel level, File saveRootDir) {
private final ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread");
private final ConcurrentHashMap<DhSectionPos, RenderMetaDataFile> files = new ConcurrentHashMap<>();
private final IDhClientLevel level;
private final File saveDir;
private final IDataSourceProvider dataSourceProvider;
public RenderFileHandler(IDataSourceProvider sourceProvider, IDhClientLevel level, File saveRootDir)
{
this.dataSourceProvider = sourceProvider;
this.level = level;
this.saveDir = saveRootDir;
}
/*
/**
* Caller must ensure that this method is called only once,
* and that this object is not used before this method is called.
* and that the given files are not used before this method is called.
*/
@Override
public void addScannedFile(Collection<File> detectedFiles) {
HashMultimap<DhSectionPos, RenderMetaFile> filesByPos = HashMultimap.create();
{ // Sort files by pos.
for (File file : detectedFiles) {
try {
RenderMetaFile metaFile = new RenderMetaFile(this, file);
filesByPos.put(metaFile.pos, metaFile);
} catch (IOException e) {
LOGGER.error("Failed to read render meta file at {}: ", file, e);
File corruptedFile = new File(file.getParentFile(), file.getName() + ".corrupted");
if (corruptedFile.exists()) corruptedFile.delete();
if (file.renameTo(corruptedFile)) {
LOGGER.error("Renamed corrupted file to {}", file.getName() + ".corrupted");
} else {
LOGGER.error("Failed to rename corrupted file to {}. Will try and delete file", file.getName() + ".corrupted");
file.delete();
}
}
}
}
// Warn for multiple files with the same pos, and then select the one with the latest timestamp.
for (DhSectionPos pos : filesByPos.keySet()) {
Collection<RenderMetaFile> metaFiles = filesByPos.get(pos);
RenderMetaFile fileToUse;
if (metaFiles.size() > 1) {
fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get()));
{
StringBuilder sb = new StringBuilder();
sb.append("Multiple files with the same pos: ");
sb.append(pos);
sb.append("\n");
for (RenderMetaFile metaFile : metaFiles) {
sb.append("\t");
sb.append(metaFile.path);
sb.append("\n");
}
sb.append("\tUsing: ");
sb.append(fileToUse.path);
sb.append("\n");
sb.append("(Other files will be renamed by appending \".old\" to their name.)");
LOGGER.warn(sb.toString());
// Rename all other files with the same pos to .old
for (RenderMetaFile metaFile : metaFiles) {
if (metaFile == fileToUse) continue;
File oldFile = new File(metaFile.path + ".old");
try {
if (!metaFile.path.renameTo(oldFile)) throw new RuntimeException("Renaming failed");
} catch (Exception e) {
LOGGER.error("Failed to rename file: " + metaFile.path + " to " + oldFile, e);
}
}
}
} else {
fileToUse = metaFiles.iterator().next();
}
// Add file to the list of files.
files.put(pos, fileToUse);
}
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
public void addScannedFile(Collection<File> newRenderFiles)
{
HashMultimap<DhSectionPos, RenderMetaDataFile> filesByPos = HashMultimap.create();
// Sort files by pos.
for (File file : newRenderFiles)
{
try
{
RenderMetaDataFile metaFile = new RenderMetaDataFile(this, file);
filesByPos.put(metaFile.pos, metaFile);
}
catch (IOException e)
{
LOGGER.error("Failed to read render meta file at [{}]. Error: ", file, e);
String corruptedFileName = file.getName() + ".corrupted";
File corruptedFile = new File(file.getParentFile(), corruptedFileName);
if (corruptedFile.exists())
{
// could happen if there was a corrupted file before that was removed
corruptedFile.delete();
}
if (file.renameTo(corruptedFile))
{
LOGGER.error("Renamed corrupted file to [{}].", file.getName() + ".corrupted");
}
else
{
LOGGER.error("Failed to rename corrupted file to [{}]. Attempting to delete file...", corruptedFileName);
if (!file.delete())
{
LOGGER.error("Unable to delete corrupted file [{}].", corruptedFileName);
}
}
}
}
// Warn for multiple files with the same pos, and then select the one with the latest timestamp.
for (DhSectionPos pos : filesByPos.keySet())
{
Collection<RenderMetaDataFile> metaFiles = filesByPos.get(pos);
RenderMetaDataFile fileToUse;
if (metaFiles.size() > 1)
{
fileToUse = Collections.max(metaFiles, Comparator.comparingLong(renderMetaDataFile ->
renderMetaDataFile.metaData.dataVersion.get()));
{
StringBuilder sb = new StringBuilder();
sb.append("Multiple files with the same pos: ");
sb.append(pos);
sb.append("\n");
for (RenderMetaDataFile metaFile : metaFiles)
{
sb.append("\t");
sb.append(metaFile.path);
sb.append("\n");
}
sb.append("\tUsing: ");
sb.append(fileToUse.path);
sb.append("\n");
sb.append("(Other files will be renamed by appending \".old\" to their name.)");
LOGGER.warn(sb.toString());
// Rename all other files with the same pos to .old
for (RenderMetaDataFile metaFile : metaFiles)
{
if (metaFile == fileToUse)
continue;
File oldFile = new File(metaFile.path + ".old");
try
{
if (!metaFile.path.renameTo(oldFile))
throw new RuntimeException("Renaming failed");
}
catch (Exception e)
{
LOGGER.error("Failed to rename file: [" + metaFile.path + "] to [" + oldFile + "]", e);
}
}
}
}
else
{
fileToUse = metaFiles.iterator().next();
}
// Add this file to the list of files.
this.files.put(pos, fileToUse);
}
}
/** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */
@Override
public CompletableFuture<ILodRenderSource> read(DhSectionPos pos) {
RenderMetaFile metaFile = files.get(pos);
if (metaFile == null) {
RenderMetaFile newMetaFile;
try {
newMetaFile = new RenderMetaFile(this, pos);
} catch (IOException e) {
LOGGER.error("IOException on creating new render file at {}", pos, e);
return null;
}
metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
if (metaFile == null) metaFile = newMetaFile;
}
return metaFile.loadOrGetCached(renderCacheThread, level).handle(
(render, e) -> {
if (e != null) {
LOGGER.error("Uncaught error on {}:", pos, e);
}
if (render != null) return render;
return new PlaceHolderRenderSource(pos);
}
public CompletableFuture<ILodRenderSource> read(DhSectionPos pos)
{
RenderMetaDataFile metaFile = this.files.get(pos);
if (metaFile == null)
{
RenderMetaDataFile newMetaFile;
try
{
newMetaFile = new RenderMetaDataFile(this, pos);
}
catch (IOException e)
{
LOGGER.error("IOException on creating new render file at {}", pos, e);
return null;
}
metaFile = this.files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
if (metaFile == null)
{
metaFile = newMetaFile;
}
}
return metaFile.loadOrGetCached(this.renderCacheThread, this.level).handle(
(renderSource, exception) ->
{
if (exception != null)
{
LOGGER.error("Uncaught error on {}:", pos, exception);
}
return (renderSource != null) ? renderSource : new PlaceHolderRenderSource(pos);
}
);
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
/* This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */
@Override
public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) {
if (chunkData.getBBoxLodPos().convertUpwardsTo((byte)6).equals(new DhLodPos((byte)6, 10, -11))) {
public void write(DhSectionPos sectionPos, ChunkSizedData chunkData)
{
// can be used for debugging
if (chunkData.getBBoxLodPos().convertUpwardsTo((byte)6).equals(new DhLodPos((byte)6, 10, -11)))
{
int doNothing = 0;
}
recursive_write(sectionPos,chunkData);
dataSourceProvider.write(sectionPos, chunkData);
this.writeRecursively(sectionPos,chunkData);
this.dataSourceProvider.write(sectionPos, chunkData);
}
private void recursive_write(DhSectionPos sectPos, ChunkSizedData chunkData) {
if (!sectPos.getSectionBBoxPos().overlaps(new DhLodPos((byte) (4 + chunkData.dataDetail), chunkData.x, chunkData.z))) return;
if (sectPos.sectionDetail > ColumnRenderSource.SECTION_SIZE_OFFSET) {
recursive_write(sectPos.getChildByIndex(0), chunkData);
recursive_write(sectPos.getChildByIndex(1), chunkData);
recursive_write(sectPos.getChildByIndex(2), chunkData);
recursive_write(sectPos.getChildByIndex(3), chunkData);
}
RenderMetaFile metaFile = files.get(sectPos);
if (metaFile != null) { // Fast path: if there is a file for this section, just write to it.
metaFile.updateChunkIfNeeded(chunkData, level);
}
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
private void writeRecursively(DhSectionPos sectPos, ChunkSizedData chunkData)
{
if (!sectPos.getSectionBBoxPos().overlaps(new DhLodPos((byte) (4 + chunkData.dataDetail), chunkData.x, chunkData.z)))
{
return;
}
if (sectPos.sectionDetail > ColumnRenderSource.SECTION_SIZE_OFFSET)
{
this.writeRecursively(sectPos.getChildByIndex(0), chunkData);
this.writeRecursively(sectPos.getChildByIndex(1), chunkData);
this.writeRecursively(sectPos.getChildByIndex(2), chunkData);
this.writeRecursively(sectPos.getChildByIndex(3), chunkData);
}
RenderMetaDataFile metaFile = this.files.get(sectPos);
// Fast path: if there is a file for this section, just write to it.
if (metaFile != null)
{
metaFile.updateChunkIfNeeded(chunkData, this.level);
}
}
/** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */
@Override
public CompletableFuture<Void> flushAndSave() {
ArrayList<CompletableFuture<Void>> futures = new ArrayList<CompletableFuture<Void>>();
for (RenderMetaFile metaFile : files.values()) {
futures.add(metaFile.flushAndSave(renderCacheThread));
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
private File computeDefaultFilePath(DhSectionPos pos) { //TODO: Temp code as we haven't decided on the file naming & location yet.
return new File(saveDir, pos.serialize() + ".lod");
public CompletableFuture<Void> flushAndSave()
{
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
for (RenderMetaDataFile metaFile : this.files.values())
{
futures.add(metaFile.flushAndSave(this.renderCacheThread));
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
private File computeDefaultFilePath(DhSectionPos pos)
{
//TODO: Temp code as we haven't decided on the file naming & location yet.
return new File(this.saveDir, pos.serialize() + ".lod");
}
@Override
public void close() {
ArrayList<CompletableFuture<Void>> futures = new ArrayList<CompletableFuture<Void>>();
for (RenderMetaFile metaFile : files.values()) {
futures.add(metaFile.flushAndSave(renderCacheThread));
public void close()
{
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
for (RenderMetaDataFile metaFile : this.files.values())
{
futures.add(metaFile.flushAndSave(this.renderCacheThread));
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
}
public File computeRenderFilePath(DhSectionPos pos) {
return new File(saveDir, pos.serialize() + ".lod");
public File computeRenderFilePath(DhSectionPos pos)
{
return new File(this.saveDir, pos.serialize() + ".lod");
}
public CompletableFuture<ILodRenderSource> onCreateRenderFile(RenderMetaFile file) {
final int vertSize = Config.Client.Graphics.Quality.verticalQuality
.get().calculateMaxVerticalData((byte) (file.pos.sectionDetail - ColumnRenderSource.SECTION_SIZE_OFFSET));
return CompletableFuture.completedFuture(
new ColumnRenderSource(file.pos, vertSize, level.getMinY()));
}
public CompletableFuture<ILodRenderSource> onCreateRenderFile(RenderMetaDataFile file)
{
final int vertSize = Config.Client.Graphics.Quality.verticalQuality
.get().calculateMaxVerticalData((byte) (file.pos.sectionDetail - ColumnRenderSource.SECTION_SIZE_OFFSET));
return CompletableFuture.completedFuture(
new ColumnRenderSource(file.pos, vertSize, this.level.getMinY()));
}
private final ConcurrentHashMap<DhSectionPos, Object> cacheRecreationGuards = new ConcurrentHashMap<>();
private void updateCache(ILodRenderSource data, RenderMetaFile file) {
if (cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null) return;
final WeakReference<ILodRenderSource> dataRef = new WeakReference<>(data);
CompletableFuture<ILodDataSource> dataFuture = dataSourceProvider.read(data.getSectionPos());
dataFuture = dataFuture.thenApply((d) -> {
if (dataRef.get() == null) throw new UncheckedInterruptedException();
LodUtil.assertTrue(d != null);
return d;
}).exceptionally((ex) -> {
if (ex != null)
LOGGER.error("Uncaught exception when getting data for updateCache()", ex);
return null;
});
private void updateCache(ILodRenderSource data, RenderMetaDataFile file)
{
if (this.cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null)
{
return;
}
final WeakReference<ILodRenderSource> dataRef = new WeakReference<>(data);
CompletableFuture<ILodDataSource> dataFuture = this.dataSourceProvider.read(data.getSectionPos());
dataFuture = dataFuture.thenApply((dataSource) ->
{
if (dataRef.get() == null)
throw new UncheckedInterruptedException();
LodUtil.assertTrue(dataSource != null);
return dataSource;
}).exceptionally((ex) ->
{
if (ex != null)
{
LOGGER.error("Uncaught exception when getting data for updateCache()", ex);
}
return null;
});
LOGGER.info("Recreating cache for {}", data.getSectionPos());
DataRenderTransformer.asyncTransformDataSource(dataFuture, this.level)
.thenAccept((newRenderDataSource) -> this.write(dataRef.get(), file, newRenderDataSource, this.dataSourceProvider.getCacheVersion(data.getSectionPos())))
.exceptionally((ex) -> {
if (!UncheckedInterruptedException.isThrowableInterruption(ex))
LOGGER.error("Exception when updating render file using data source: ", ex);
return null;
}).thenRun(() -> this.cacheRecreationGuards.remove(file.pos));
}
LOGGER.info("Recreating cache for {}", data.getSectionPos());
DataRenderTransformer.asyncTransformDataSource(dataFuture , level)
.thenAccept((newData) -> write(dataRef.get(), file, newData, dataSourceProvider.getCacheVersion(data.getSectionPos())))
.exceptionally((ex) -> {
if (!UncheckedInterruptedException.isThrowableInterruption(ex))
LOGGER.error("Exception when updating render file using data source: ", ex);
return null;
}).thenRun(() -> cacheRecreationGuards.remove(file.pos));
}
public ILodRenderSource onRenderFileLoaded(ILodRenderSource data, RenderMetaFile file) {
if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) {
updateCache(data, file);
public ILodRenderSource onRenderFileLoaded(ILodRenderSource data, RenderMetaDataFile file)
{
if (!this.dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get()))
{
this.updateCache(data, file);
}
return data;
}
public ILodRenderSource onLoadingRenderFile(RenderMetaFile file) {
return null; //Default behaviour
public ILodRenderSource onLoadingRenderFile(RenderMetaDataFile file)
{
return null; // Default behavior: do nothing
}
private void write(ILodRenderSource target, RenderMetaFile file,
ILodRenderSource newData, long newDataVersion) {
if (target == null) return;
if (newData == null) return;
private void write(ILodRenderSource target, RenderMetaDataFile file,
ILodRenderSource newData, long newDataVersion)
{
if (target == null || newData == null)
{
return;
}
target.updateFromRenderSource(newData);
file.metaData.dataVersion.set(newDataVersion);
file.metaData.dataLevel = target.getDataDetail();
@@ -247,29 +332,37 @@ public class RenderFileHandler implements IRenderSourceProvider {
file.dataType = target.getClass();
file.metaData.dataTypeId = file.loader.renderTypeId;
file.metaData.loaderVersion = target.getRenderVersion();
file.save(target, level);
file.save(target, this.level);
}
public void onReadRenderSourceFromCache(RenderMetaFile file, ILodRenderSource data) {
if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) {
updateCache(data, file);
public void onReadRenderSourceFromCache(RenderMetaDataFile file, ILodRenderSource data)
{
if (!this.dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get()))
{
this.updateCache(data, file);
}
}
public boolean refreshRenderSource(ILodRenderSource source) {
RenderMetaFile file = files.get(source.getSectionPos());
if (source instanceof PlaceHolderRenderSource) {
if (file == null || file.metaData == null) {
public boolean refreshRenderSource(ILodRenderSource source)
{
RenderMetaDataFile file = this.files.get(source.getSectionPos());
if (source instanceof PlaceHolderRenderSource)
{
if (file == null || file.metaData == null)
{
return false;
}
}
LodUtil.assertTrue(file != null);
LodUtil.assertTrue(file.metaData != null);
if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) {
updateCache(source, file);
if (!this.dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get()))
{
this.updateCache(source, file);
return true;
}
return false;
}
}
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java
+289
View File
@@ -0,0 +1,289 @@
package com.seibel.lod.core.file.renderfile;
import com.seibel.lod.core.datatype.ILodRenderSource;
import com.seibel.lod.core.datatype.AbstractRenderSourceLoader;
import com.seibel.lod.core.datatype.full.ChunkSizedData;
import com.seibel.lod.core.file.metaData.MetaData;
import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.file.metaData.MetaDataFile;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.ref.SoftReference;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
public class RenderMetaDataFile extends MetaDataFile
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public AbstractRenderSourceLoader loader;
public Class<? extends ILodRenderSource> dataType;
// The '?' type should either be:
// SoftReference<LodRenderSource>, or - File that may still be loaded
// CompletableFuture<LodRenderSource>,or - File that is being loaded
// null - Nothing is loaded or being loaded
AtomicReference<Object> data = new AtomicReference<>(null);
// @FunctionalInterface
// public interface CacheValidator
// {
// boolean isCacheValid(DhSectionPos sectionPos, long timestamp);
// }
// @FunctionalInterface
// public interface CacheSourceProducer
// {
// CompletableFuture<ILodDataSource> getSourceFuture(DhSectionPos sectionPos);
// }
// CacheValidator validator;
// CacheSourceProducer source;
private final RenderFileHandler fileHandler;
private boolean doesFileExist;
/** Creates a new metaFile */
public RenderMetaDataFile(RenderFileHandler fileHandler, DhSectionPos pos) throws IOException
{
super(fileHandler.computeRenderFilePath(pos), pos);
this.fileHandler = fileHandler;
LodUtil.assertTrue(this.metaData == null);
this.doesFileExist = false;
}
/** Uses the existing metaFile */
public RenderMetaDataFile(RenderFileHandler fileHandler, File path) throws IOException
{
super(path);
this.fileHandler = fileHandler;
LodUtil.assertTrue(this.metaData != null);
this.loader = AbstractRenderSourceLoader.getLoader(this.metaData.dataTypeId, this.metaData.loaderVersion);
if (this.loader == null)
{
throw new IOException("Invalid file: Data type loader not found: "
+ this.metaData.dataTypeId + "(v" + this.metaData.loaderVersion + ")");
}
this.dataType = this.loader.clazz;
this.doesFileExist = true;
}
// FIXME: This can cause concurrent modification of LodRenderSource.
// Not sure if it will cause issues or not.
public void updateChunkIfNeeded(ChunkSizedData chunkData, IDhClientLevel level)
{
DhLodPos chunkPos = new DhLodPos((byte) (chunkData.dataDetail + 4), chunkData.x, chunkData.z);
LodUtil.assertTrue(this.pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos {} doesn't overlap with section {}", chunkPos, pos);
CompletableFuture<ILodRenderSource> source = this._readCached(this.data.get());
if (source == null)
{
return;
}
source.thenAccept((renderSource) -> renderSource.fastWrite(chunkData, level));
}
public CompletableFuture<Void> flushAndSave(ExecutorService renderCacheThread)
{
if (!path.exists())
{
return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist.
}
CompletableFuture<ILodRenderSource> source = this._readCached(this.data.get());
if (source == null)
{
return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save.
}
return source.thenAccept((a) -> { }); // Otherwise, wait for the data to be read (which also flushes changes to the file).
}
// Suppress casting of CompletableFuture<?> to CompletableFuture<LodRenderSource>
@SuppressWarnings("unchecked")
private CompletableFuture<ILodRenderSource> _readCached(Object obj)
{
// Has file cached in RAM and not freed yet.
if ((obj instanceof SoftReference<?>))
{
Object inner = ((SoftReference<?>) obj).get();
if (inner != null)
{
LodUtil.assertTrue(inner instanceof ILodRenderSource);
fileHandler.onReadRenderSourceFromCache(this, (ILodRenderSource) inner);
return CompletableFuture.completedFuture((ILodRenderSource) inner);
}
}
//==== Cached file out of scope. ====
// Someone is already trying to complete it. so just return the obj.
if ((obj instanceof CompletableFuture<?>))
{
return (CompletableFuture<ILodRenderSource>) obj;
}
return null;
}
// Cause: Generic Type runtime casting cannot safety check it.
// However, the Union type ensures the 'data' should only contain the listed type.
public CompletableFuture<ILodRenderSource> loadOrGetCached(Executor fileReaderThreads, IDhLevel level)
{
Object obj = this.data.get();
CompletableFuture<ILodRenderSource> cached = this._readCached(obj);
if (cached != null)
{
return cached;
}
// Create an empty and non-completed future.
// Note: I do this before actually filling in the future so that I can ensure only
// one task is submitted to the thread pool.
CompletableFuture<ILodRenderSource> future = new CompletableFuture<>();
// Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :(
boolean worked = this.data.compareAndSet(obj, future);
if (!worked)
{
return this.loadOrGetCached(fileReaderThreads, level);
}
// Now, there should only ever be one thread at a time here due to the CAS operation above.
// After cas. We are in exclusive control.
if (!this.doesFileExist)
{
this.fileHandler.onCreateRenderFile(this)
.thenApply((data) ->
{
this.metaData = makeMetaData(data);
return data;
})
.thenApply((d) -> this.fileHandler.onRenderFileLoaded(d, this))
.whenComplete((v, e) ->
{
if (e != null)
{
LOGGER.error("Uncaught error on creation {}: ", this.path, e);
future.complete(null);
this.data.set(null);
}
else
{
future.complete(v);
//new DataObjTracker(v); //TODO: Obj Tracker??? For debug?
this.data.set(new SoftReference<>(v));
}
});
}
else
{
CompletableFuture.supplyAsync(() ->
{
if (this.metaData == null)
{
throw new IllegalStateException("Meta data not loaded!");
}
// Load the file.
ILodRenderSource data;
data = this.fileHandler.onLoadingRenderFile(this);
if (data == null)
{
try (FileInputStream fio = getDataContent())
{
data = this.loader.loadRender(this, fio, level);
}
catch (IOException e)
{
throw new CompletionException(e);
}
}
data = this.fileHandler.onRenderFileLoaded(data, this);
return data;
}, fileReaderThreads)
.whenComplete((f, e) ->
{
if (e != null)
{
LOGGER.error("Error loading file {}: ", this.path, e);
future.complete(null);
this.data.set(null);
}
else
{
future.complete(f);
this.data.set(new SoftReference<>(f));
}
});
}
return future;
}
private static MetaData makeMetaData(ILodRenderSource data)
{
AbstractRenderSourceLoader loader = AbstractRenderSourceLoader.getLoader(data.getClass(), data.getRenderVersion());
return new MetaData(data.getSectionPos(), -1, -1,
data.getDataDetail(), loader == null ? 0 : loader.renderTypeId, data.getRenderVersion());
}
private FileInputStream getDataContent() throws IOException
{
FileInputStream fin = new FileInputStream(this.path);
int toSkip = METADATA_SIZE;
while (toSkip > 0)
{
long skipped = fin.skip(toSkip);
if (skipped == 0)
{
throw new IOException("Invalid file: Failed to skip metadata.");
}
toSkip -= skipped;
}
if (toSkip != 0)
{
throw new IOException("File IO Error: Failed to skip metadata.");
}
return fin;
}
public void save(ILodRenderSource data, IDhClientLevel level)
{
if (data.isEmpty())
{
if (this.path.exists())
{
if (!this.path.delete())
{
LOGGER.warn("Failed to delete render file at {}", this.path);
}
}
this.doesFileExist = false;
}
else
{
LOGGER.info("Saving updated render file v[{}] at sect {}", this.metaData.dataVersion.get(), this.pos);
try
{
super.writeData((out) -> data.saveRender(level, this, out));
this.doesFileExist = true;
}
catch (IOException e)
{
LOGGER.error("Failed to save updated render file at {} for sect {}", this.path, this.pos, e);
}
}
}
}
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaFile.java
-218
View File
@@ -1,218 +0,0 @@
package com.seibel.lod.core.file.renderfile;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.ILodRenderSource;
import com.seibel.lod.core.datatype.AbstractRenderSourceLoader;
import com.seibel.lod.core.datatype.full.ChunkSizedData;
import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.file.subDimMatching.MetaFile;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.ref.SoftReference;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
public class RenderMetaFile extends MetaFile
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger(RenderMetaFile.class.getSimpleName());
//private final IClientLevel level;
public AbstractRenderSourceLoader loader;
public Class<? extends ILodRenderSource> dataType;
// The '?' type should either be:
// SoftReference<LodRenderSource>, or - File that may still be loaded
// CompletableFuture<LodRenderSource>,or - File that is being loaded
// null - Nothing is loaded or being loaded
AtomicReference<Object> data = new AtomicReference<>(null);
//FIXME: This can cause concurrent modification of LodRenderSource.
// Not sure if it will cause issues or not.
public void updateChunkIfNeeded(ChunkSizedData chunkData, IDhClientLevel level) {
DhLodPos chunkPos = new DhLodPos((byte) (chunkData.dataDetail + 4), chunkData.x, chunkData.z);
LodUtil.assertTrue(pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos {} doesn't overlap with section {}", chunkPos, pos);
CompletableFuture<ILodRenderSource> source = _readCached(data.get());
if (source == null) return;
source.thenAccept((renderSource) -> renderSource.fastWrite(chunkData, level));
}
public CompletableFuture<Void> flushAndSave(ExecutorService renderCacheThread) {
if (!path.exists()) return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist.
CompletableFuture<ILodRenderSource> source = _readCached(data.get());
if (source == null) return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save.
return source.thenAccept((a)->{}); // Otherwise, wait for the data to be read (which also flushes changes to the file).
}
@FunctionalInterface
public interface CacheValidator {
boolean isCacheValid(DhSectionPos sectionPos, long timestamp);
}
@FunctionalInterface
public interface CacheSourceProducer {
CompletableFuture<ILodDataSource> getSourceFuture(DhSectionPos sectionPos);
}
CacheValidator validator;
CacheSourceProducer source;
final RenderFileHandler handler;
private boolean doesFileExist;
// Create a new metaFile
public RenderMetaFile(RenderFileHandler handler, DhSectionPos pos) throws IOException {
super(handler.computeRenderFilePath(pos), pos);
this.handler = handler;
LodUtil.assertTrue(metaData == null);
doesFileExist = false;
}
public RenderMetaFile(RenderFileHandler handler, File path) throws IOException {
super(path);
this.handler = handler;
LodUtil.assertTrue(metaData != null);
loader = AbstractRenderSourceLoader.getLoader(metaData.dataTypeId, metaData.loaderVersion);
if (loader == null) {
throw new IOException("Invalid file: Data type loader not found: "
+ metaData.dataTypeId + "(v" + metaData.loaderVersion + ")");
}
dataType = loader.clazz;
doesFileExist = true;
}
// Suppress casting of CompletableFuture<?> to CompletableFuture<LodRenderSource>
@SuppressWarnings("unchecked")
private CompletableFuture<ILodRenderSource> _readCached(Object obj) {
// Has file cached in RAM and not freed yet.
if ((obj instanceof SoftReference<?>)) {
Object inner = ((SoftReference<?>)obj).get();
if (inner != null) {
LodUtil.assertTrue(inner instanceof ILodRenderSource);
handler.onReadRenderSourceFromCache(this, (ILodRenderSource) inner);
return CompletableFuture.completedFuture((ILodRenderSource)inner);
}
}
//==== Cached file out of scope. ====
// Someone is already trying to complete it. so just return the obj.
if ((obj instanceof CompletableFuture<?>)) {
return (CompletableFuture<ILodRenderSource>)obj;
}
return null;
}
// Cause: Generic Type runtime casting cannot safety check it.
// However, the Union type ensures the 'data' should only contain the listed type.
public CompletableFuture<ILodRenderSource> loadOrGetCached(Executor fileReaderThreads, IDhLevel level) {
Object obj = data.get();
CompletableFuture<ILodRenderSource> cached = _readCached(obj);
if (cached != null) return cached;
// Create an empty and non-completed future.
// Note: I do this before actually filling in the future so that I can ensure only
// one task is submitted to the thread pool.
CompletableFuture<ILodRenderSource> future = new CompletableFuture<>();
// Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :(
boolean worked = data.compareAndSet(obj, future);
if (!worked) return loadOrGetCached(fileReaderThreads, level);
// Now, there should only ever be one thread at a time here due to the CAS operation above.
// After cas. We are in exclusive control.
if (!doesFileExist) {
handler.onCreateRenderFile(this)
.thenApply((data) -> {
metaData = makeMetaData(data);
return data;
})
.thenApply((d) -> handler.onRenderFileLoaded(d, this))
.whenComplete((v, e) -> {
if (e != null) {
LOGGER.error("Uncaught error on creation {}: ", path, e);
future.complete(null);
data.set(null);
} else {
future.complete(v);
//new DataObjTracker(v); //TODO: Obj Tracker??? For debug?
data.set(new SoftReference<>(v));
}
});
} else {
CompletableFuture.supplyAsync(() -> {
if (metaData == null)
throw new IllegalStateException("Meta data not loaded!");
// Load the file.
ILodRenderSource data;
data = handler.onLoadingRenderFile(this);
if (data == null) {
try (FileInputStream fio = getDataContent()) {
data = loader.loadRender(this, fio, level);
} catch (IOException e) {
throw new CompletionException(e);
}
}
data = handler.onRenderFileLoaded(data, this);
return data;
}, fileReaderThreads)
.whenComplete((f, e) -> {
if (e != null) {
LOGGER.error("Error loading file {}: ", path, e);
future.complete(null);
data.set(null);
} else {
future.complete(f);
data.set(new SoftReference<>(f));
}
});
}
return future;
}
private static MetaData makeMetaData(ILodRenderSource data) {
AbstractRenderSourceLoader loader = AbstractRenderSourceLoader.getLoader(data.getClass(), data.getRenderVersion());
return new MetaData(data.getSectionPos(), -1, -1,
data.getDataDetail(), loader == null ? 0 : loader.renderTypeId, data.getRenderVersion());
}
private FileInputStream getDataContent() throws IOException {
FileInputStream fin = new FileInputStream(path);
int toSkip = METADATA_SIZE;
while (toSkip > 0) {
long skipped = fin.skip(toSkip);
if (skipped == 0) {
throw new IOException("Invalid file: Failed to skip metadata.");
}
toSkip -= skipped;
}
if (toSkip != 0) {
throw new IOException("File IO Error: Failed to skip metadata.");
}
return fin;
}
public void save(ILodRenderSource data, IDhClientLevel level) {
if (data.isEmpty()) {
if (path.exists()) if (!path.delete()) LOGGER.warn("Failed to delete render file at {}", path);
doesFileExist = false;
} else {
LOGGER.info("Saving updated render file v[{}] at sect {}", metaData.dataVersion.get(), pos);
try {
super.writeData((out) -> data.saveRender(level, this, out));
doesFileExist = true;
} catch (IOException e) {
LOGGER.error("Failed to save updated render file at {} for sect {}", path, pos, e);
}
}
}
}
core/src/main/java/com/seibel/lod/core/file/subDimMatching/MetaFile.java
-197
View File
@@ -1,197 +0,0 @@
package com.seibel.lod.core.file.subDimMatching;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.objects.UnclosableOutputStream;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
/**
* Used size: 40 bytes <br>
* Remaining space: 24 bytes <br>
* Total size: 64 bytes <br> <br><br>
*
*
* Metadata format: <br> <br>
*
* 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signals the metadata format) <br>
* 4 bytes: section X position <br>
* 4 bytes: section Y position (Unused, for future proofing) <br>
* 4 bytes: section Z position <br> <br>
*
* 4 bytes: data checksum <br> //TODO: Implement checksum
* 1 byte: section detail level <br>
* 1 byte: data detail level // Note: not sure if this is needed <br>
* 1 byte: loader version <br>
* 1 byte: unused <br> <br>
*
* 8 bytes: datatype identifier <br> <br>
*
* 8 bytes: data version
*/
public class MetaFile
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final int METADATA_SIZE = 64;
public static final int METADATA_RESERVED_SIZE = 24;
public static final int METADATA_MAGIC_BYTES = 0x44_48_76_30;
/** Currently set to false because for some reason Window is throwing PermissionDeniedException when trying to atomic replace a file... */
public static final boolean USE_ATOMIC_MOVE_REPLACE = false;
public final DhSectionPos pos;
public File path;
@FunctionalInterface
public interface IOConsumer<T> {
void accept(T t) throws IOException;
}
public static class MetaData {
public DhSectionPos pos;
public int checksum;
public AtomicLong dataVersion;
public byte dataLevel;
//Loader stuff
public long dataTypeId;
public byte loaderVersion;
public MetaData(DhSectionPos pos, int checksum, long dataVersion, byte dataLevel, long dataTypeId, byte loaderVersion) {
this.pos = pos;
this.checksum = checksum;
this.dataVersion = new AtomicLong(dataVersion);
this.dataLevel = dataLevel;
this.dataTypeId = dataTypeId;
this.loaderVersion = loaderVersion;
}
}
public volatile MetaData metaData = null;
private static MetaData readMeta(File file) throws IOException {
try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE);
channel.read(buffer, 0);
channel.close();
buffer.flip();
int magic = buffer.getInt();
if (magic != METADATA_MAGIC_BYTES) {
throw new IOException("Invalid file: Magic bytes check failed.");
}
int x = buffer.getInt();
int y = buffer.getInt(); // Unused
int z = buffer.getInt();
int checksum = buffer.getInt();
byte detailLevel = buffer.get();
byte dataLevel = buffer.get();
byte loaderVersion = buffer.get();
byte unused = buffer.get();
long dataTypeId = buffer.getLong();
long timestamp = buffer.getLong();
LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE);
DhSectionPos dataPos = new DhSectionPos(detailLevel, x, z);
return new MetaData(dataPos, checksum, timestamp, dataLevel, dataTypeId, loaderVersion);
}
}
private void validateFile() throws IOException {
if (!path.exists()) throw new IOException("File missing");
if (!path.isFile()) throw new IOException("Not a file");
if (!path.canRead()) throw new IOException("File not readable");
if (!path.canWrite()) throw new IOException("File not writable");
}
// Create a metaFile in this path. If the path has a file, throws FileAlreadyExistsException
protected MetaFile(File path, DhSectionPos pos) throws IOException {
this.path = path;
this.pos = pos;
if (path.exists()) throw new FileAlreadyExistsException(path.toString());
}
// Load a metaFile in this path
protected MetaFile(File path) throws IOException {
this.path = path;
if (!path.exists()) throw new FileNotFoundException("File not found at " + path);
validateFile();
metaData = readMeta(path);
pos = metaData.pos;
}
protected void loadMetaData() throws IOException {
validateFile();
metaData = readMeta(path);
if (!metaData.pos.equals(pos)) {
LOGGER.warn("The file is from a different location than expected! Expects {} but got {}. Ignoring file tag.", pos, metaData.pos);
metaData.pos = pos;
}
}
protected void writeData(IOConsumer<OutputStream> dataWriter) throws IOException {
LodUtil.assertTrue(metaData != null);
if (path.exists()) validateFile();
File writerFile;
if (USE_ATOMIC_MOVE_REPLACE) {
writerFile = new File(path.getPath() + ".tmp");
writerFile.deleteOnExit();
} else {
writerFile = path;
}
try (FileChannel file = FileChannel.open(writerFile.toPath(),
StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
{
file.position(METADATA_SIZE);
int checksum;
try (OutputStream channelOut = new UnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel
BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer?
CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) { // TODO: Is Adler32 ok?
dataWriter.accept(checkedOut);
checksum = (int) checkedOut.getChecksum().getValue();
}
file.position(0);
// Write metadata
ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE);
buff.putInt(METADATA_MAGIC_BYTES);
buff.putInt(pos.sectionX);
buff.putInt(Integer.MIN_VALUE); // Unused
buff.putInt(pos.sectionZ);
buff.putInt(checksum);
buff.put(pos.sectionDetail);
buff.put(metaData.dataLevel);
buff.put(metaData.loaderVersion);
buff.put(Byte.MIN_VALUE); // Unused
buff.putLong(metaData.dataTypeId);
buff.putLong(metaData.dataVersion.get());
LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE);
buff.flip();
file.write(buff);
}
file.close();
if (USE_ATOMIC_MOVE_REPLACE) {
// Atomic move / replace the actual file
Files.move(writerFile.toPath(), path.toPath(), StandardCopyOption.ATOMIC_MOVE);
}
} finally {
try {
if (USE_ATOMIC_MOVE_REPLACE && writerFile.exists()) {
boolean i = writerFile.delete(); // Delete temp file. Ignore errors if fails.
}
} catch (Exception ignored) {}
}
}
}