wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://gitlab.com/jeseibel/distant-horizons-core

Browse Source
This commit is contained in:
James Seibel
2022-06-16 19:57:41 -05:00
parent b13559b172 fd58b10800
commit 602d2c4016
15 changed files with 659 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/seibel/lod/core/objects/a7/data/DataFile.java
-183
View File
@@ -1,183 +0,0 @@
package com.seibel.lod.core.objects.a7.data;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.datatype.column.DataSourceSaver;
import com.seibel.lod.core.objects.a7.datatype.column.OldDataSourceLoader;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import java.io.*;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
public class DataFile {
//Metadata format:
//
// 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signal the metadata format)
// 4 bytes: section X position
// 4 bytes: section Y position (Unused, for future proofing)
// 4 bytes: section Z position
//
// 4 bytes: data checksum //TODO: Implement checksum
// 1 byte: section detail level
// 1 byte: data detail level // Note: not sure if this is needed
// 1 byte: loader version
// 1 byte: unused
//
// 8 bytes: datatype identifier
//
// 8 bytes: unused
// Total size: 32 bytes
public static final int METADATA_SIZE = 32;
public static final int METADATA_MAGIC_BYTES = 0x44_48_76_30;
public final File path;
public final DhSectionPos pos;
public byte dataLevel;
public DataSourceLoader loader;
public byte loaderVersion;
public Class<?> dataType;
public LodDataSource loadedData = null;
public static DataFile readMeta(File path) throws IOException {
try (FileInputStream fin = new FileInputStream(path)) {
MappedByteBuffer buffer = fin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, METADATA_SIZE);
return new DataFile(path, buffer);
}
}
public DataFile(File path, DataSourceLoader loader, LodDataSource loadedData) {
this.path = path;
this.pos = loadedData.getSectionPos();
this.loader = loader;
this.dataType = loader.clazz;
this.dataLevel = loadedData.getDataDetail();
this.loadedData = loadedData;
this.loaderVersion = loader.loaderSupportedVersions[loader.loaderSupportedVersions.length - 1]; // get latest version
}
DataFile(File path, MappedByteBuffer meta) throws IOException {
this.path = path;
int magic = meta.getInt();
if (magic != METADATA_MAGIC_BYTES) {
throw new IOException("Invalid file: Magic bytes check failed.");
}
int x = meta.getInt();
int y = meta.getInt(); // Unused
int z = meta.getInt();
int checksum = meta.getInt();
byte detailLevel = meta.get();
dataLevel = meta.get();
byte loaderVersion = meta.get();
byte unused = meta.get();
long dataTypeId = meta.getLong();
long unused2 = meta.getLong();
LodUtil.assertTrue(meta.remaining() == 0);
this.pos = new DhSectionPos(detailLevel, x, z);
this.loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion);
if (loader == null) {
throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
}
this.dataType = loader.clazz;
this.loaderVersion = loaderVersion;
}
public FileInputStream getDataContent() throws IOException {
FileInputStream fin = new FileInputStream(path);
int toSkip = METADATA_SIZE;
while (toSkip > 0) {
long skipped = fin.skip(toSkip);
if (skipped == 0) {
throw new IOException("Invalid file: Failed to skip metadata.");
}
toSkip -= skipped;
}
if (toSkip != 0) {
throw new IOException("File IO Error: Failed to skip metadata.");
}
return fin;
}
LodDataSource load(DHLevel level) {
if (loadedData != null) return loadedData;
try {
loadedData = loader.loadData(this, level);
return loadedData;
} catch (IOException e) {
//FIXME: Log and review this handling
return null;
}
}
public boolean verifyPath() {
return path.exists() && path.isFile() && path.canRead() && path.canWrite();
}
public void saveIfNeeded(DHLevel level, boolean freeMemory) {
if (loadedData == null) return;
if (!verifyPath()) return;
try {
save(level, freeMemory);
} catch (IOException e) {
//FIXME: Log and review this handling
}
}
public void save(DHLevel level, boolean freeMemory) throws IOException {
if (loadedData == null) throw new IllegalStateException("No data loaded");
if (!verifyPath()) throw new IOException("File path became invalid");
DataSourceSaver saver;
if (loader instanceof DataSourceSaver) saver = (DataSourceSaver) loader;
else if (loader instanceof OldDataSourceLoader) saver = ((OldDataSourceLoader) loader).getNewSaver();
else saver = null;
if (saver == null) return;
byte newDataLevel = loadedData.getDataDetail();
try (FileOutputStream fout = new FileOutputStream(path, false)) {
try (DataOutputStream out = new DataOutputStream(fout)) {
out.writeInt(METADATA_MAGIC_BYTES);
// Write x, y, z, checksum
out.writeInt(pos.sectionX);
out.writeInt(Integer.MIN_VALUE); // not used for now
out.writeInt(pos.sectionZ);
out.writeInt(Integer.MIN_VALUE); // not used for now
// Write detail level, data level, loader version
out.writeByte(pos.sectionDetail);
out.writeByte(loadedData.getDataDetail());
out.writeByte(saver.getSaverVersion());
// Write unused
out.writeByte((byte) 0);
// Write data type id
out.writeLong(saver.datatypeId);
// Write unused
out.writeLong(Long.MIN_VALUE);
// Write data
saver.saveData(level, loadedData, out);
}
}
dataLevel = newDataLevel;
loader = saver;
if (freeMemory) {
loadedData = null;
}
}
public void close(DHLevel level) {
if (loadedData != null) {
saveIfNeeded(level, true);
}
}
}
src/main/java/com/seibel/lod/core/objects/a7/data/DataSourceLoader.java
+3 -4
View File
@@ -2,9 +2,8 @@ package com.seibel.lod.core.objects.a7.data;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.io.file.DataMetaFile;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -38,14 +37,14 @@ public abstract class DataSourceLoader {
return false;
})) {
throw new IllegalArgumentException("Loader for class " + clazz + " that supports one of the version in "
+ loaderSupportedVersions + " already registered!");
+ Arrays.toString(loaderSupportedVersions) + " already registered!");
}
datatypeIdRegistry.put(datatypeId, clazz);
loaderRegistry.put(datatypeId, this);
}
// Can return null as meaning the requirement is not met
public abstract LodDataSource loadData(DataFile dataFile, DHLevel level) throws IOException;
public abstract LodDataSource loadData(DataMetaFile dataFile, InputStream data, DHLevel level) throws IOException;
public List<File> foldersToScan(File levelFolderPath) {
return Collections.emptyList();
src/main/java/com/seibel/lod/core/objects/a7/data/LodDataSource.java
+1
View File
@@ -9,4 +9,5 @@ public interface LodDataSource {
DataSourceLoader getLatestLoader();
DhSectionPos getSectionPos();
byte getDataDetail();
void setLocalVersion(int localVer);
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/Alpha6DataLoader.java
+3 -3
View File
@@ -3,6 +3,7 @@ package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.enums.config.EVerticalQuality;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.*;
import com.seibel.lod.core.objects.a7.io.file.DataMetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
@@ -20,11 +21,10 @@ public class Alpha6DataLoader extends OldDataSourceLoader implements OldFileConv
}
@Override
public LodDataSource loadData(DataFile dataFile, DHLevel level) {
public LodDataSource loadData(DataMetaFile dataFile, InputStream data, DHLevel level) {
//TODO: Add decompressor here
try (
FileInputStream fin = dataFile.getDataContent();
XZCompressorInputStream xzIn = new XZCompressorInputStream(fin);
XZCompressorInputStream xzIn = new XZCompressorInputStream(data);
DataInputStream dis = new DataInputStream(xzIn);
) {
return new OldColumnDatatype(dataFile.pos, dis, dataFile.loaderVersion, level, 1);
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/ColumnDataLoader.java
+8 -6
View File
@@ -3,9 +3,10 @@ package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.config.Config;
import com.seibel.lod.core.enums.config.EVerticalQuality;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFile;
import com.seibel.lod.core.objects.a7.data.DataFileHandler;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.io.file.DataMetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.*;
@@ -21,11 +22,10 @@ public class ColumnDataLoader extends DataSourceSaver {
}
@Override
public LodDataSource loadData(DataFile dataFile, DHLevel level) {
public LodDataSource loadData(DataMetaFile dataFile, InputStream data, DHLevel level) {
try (
FileInputStream fin = dataFile.getDataContent();
//TODO: Add decompressor here
DataInputStream dis = new DataInputStream(fin);
DataInputStream dis = new DataInputStream(data);
) {
return new ColumnDatatype(dataFile.pos, dis, dataFile.loaderVersion, level);
} catch (IOException e) {
@@ -35,9 +35,11 @@ public class ColumnDataLoader extends DataSourceSaver {
}
@Override
public void saveData(DHLevel level, LodDataSource loadedData, DataOutputStream out) throws IOException {
public void saveData(DHLevel level, LodDataSource loadedData, MetaFile file, OutputStream out) throws IOException {
//TODO: Add compressor here
((ColumnDatatype) loadedData).writeData(out);
try (DataOutputStream dos = new DataOutputStream(out)) {
((ColumnDatatype) loadedData).writeData(dos);
}
}
@Override
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/DataSourceSaver.java
+3 -1
View File
@@ -4,18 +4,20 @@ import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFileHandler;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
public abstract class DataSourceSaver extends DataSourceLoader {
public DataSourceSaver(Class<? extends LodDataSource> clazz, long datatypeId, byte[] loaderSupportedVersions) {
super(clazz, datatypeId, loaderSupportedVersions);
}
public abstract void saveData(DHLevel level, LodDataSource loadedData, DataOutputStream out) throws IOException;
public abstract void saveData(DHLevel level, LodDataSource loadedData, MetaFile file, OutputStream dataStream) throws IOException;
// generate the default file path and file name based on various parameters.
// Ensure the file extension is '.lod'!
public File generateFilePathAndName(File levelFolderPath, DHLevel level, DhSectionPos sectionPos) {
src/main/java/com/seibel/lod/core/objects/a7/io/DataSourceProvider.java
+13
View File
@@ -0,0 +1,13 @@
package com.seibel.lod.core.objects.a7.io;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.util.concurrent.CompletableFuture;
public interface DataSourceProvider {
CompletableFuture<LodDataSource> read(DhSectionPos pos);
void write(DhSectionPos sectionPos, FullDatatype chunkData);
CompletableFuture<Void> flushAndSave();
}
src/main/java/com/seibel/lod/core/objects/a7/io/FileScanner.java
+4
View File
@@ -0,0 +1,4 @@
package com.seibel.lod.core.objects.a7.io;
public class FileScanner {
}
src/main/java/com/seibel/lod/core/objects/a7/io/MetaFile.java
+181
View File
@@ -0,0 +1,181 @@
package com.seibel.lod.core.objects.a7.io;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.function.BiConsumer;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import net.fabricmc.mapping.tree.Mapped;
public class MetaFile {
//Metadata format:
//
// 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signal the metadata format)
// 4 bytes: section X position
// 4 bytes: section Y position (Unused, for future proofing)
// 4 bytes: section Z position
//
// 4 bytes: data checksum //TODO: Implement checksum
// 1 byte: section detail level
// 1 byte: data detail level // Note: not sure if this is needed
// 1 byte: loader version
// 1 byte: unused
//
// 8 bytes: datatype identifier
//
// 8 bytes: timestamp
// Total size: 32 bytes
public static final int METADATA_SIZE = 32;
public static final int METADATA_MAGIC_BYTES = 0x44_48_76_30;
public final DhSectionPos pos;
public File path;
public int checksum;
public long timestamp;
public byte dataLevel;
//Loader stuff
public DataSourceLoader loader;
public Class<?> dataType;
public byte loaderVersion;
// Load a metaFile in this path. It also automatically read the metadata.
protected MetaFile(File path) throws IOException {
validatePath();
try (FileInputStream fin = new FileInputStream(path)) {
MappedByteBuffer buffer = fin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, METADATA_SIZE);
this.path = path;
int magic = buffer.getInt();
if (magic != METADATA_MAGIC_BYTES) {
throw new IOException("Invalid file: Magic bytes check failed.");
}
int x = buffer.getInt();
int y = buffer.getInt(); // Unused
int z = buffer.getInt();
int checksum = buffer.getInt();
byte detailLevel = buffer.get();
dataLevel = buffer.get();
byte loaderVersion = buffer.get();
byte unused = buffer.get();
long dataTypeId = buffer.getLong();
long timestamp = buffer.getLong();
LodUtil.assertTrue(buffer.remaining() == 0);
this.pos = new DhSectionPos(detailLevel, x, z);
this.loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion);
if (loader == null) {
throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
}
this.dataType = loader.clazz;
this.loaderVersion = loaderVersion;
}
}
// Make a new MetaFile. It doesn't load or write any metadata itself.
protected MetaFile(File path, DhSectionPos pos) {
this.path = path;
this.pos = pos;
}
protected void save() {} //TODO: Implement
private void validatePath() throws IOException {
if (!path.exists()) throw new IOException("File missing");
if (!path.isFile()) throw new IOException("Not a file");
if (!path.canRead()) throw new IOException("File not readable");
if (!path.canWrite()) throw new IOException("File not writable");
}
protected void updateMetaData() throws IOException {
validatePath();
try (FileChannel channel = FileChannel.open(path.toPath(), StandardOpenOption.READ)) {
MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, METADATA_SIZE);
int magic = buffer.getInt();
if (magic != METADATA_MAGIC_BYTES) {
throw new IOException("Invalid file: Magic bytes check failed.");
}
int x = buffer.getInt();
int y = buffer.getInt(); // Unused
int z = buffer.getInt();
int checksum = buffer.getInt();
byte detailLevel = buffer.get();
dataLevel = buffer.get();
byte loaderVersion = buffer.get();
byte unused = buffer.get();
long dataTypeId = buffer.getLong();
long timestamp = buffer.getLong();
LodUtil.assertTrue(buffer.remaining() == 0);
DhSectionPos newPos = new DhSectionPos(detailLevel, x, z);
if (!newPos.equals(pos)) {
throw new IOException("Invalid file: Section position changed.");
}
this.loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion);
if (loader == null) {
throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
}
this.dataType = loader.clazz;
this.loaderVersion = loaderVersion;
}
}
protected void writeData(BiConsumer<MetaFile, OutputStream> dataWriter) throws IOException {
validatePath();
File tempFile = File.createTempFile("", "tmp", path.getParentFile());
tempFile.deleteOnExit();
try (FileChannel file = FileChannel.open(tempFile.toPath(),
StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
{
file.position(METADATA_SIZE);
int checksum;
try (OutputStream channelOut = Channels.newOutputStream(file);
BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer?
CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) { // TODO: Is Adler32 ok?
dataWriter.accept(this, checkedOut);
checksum = (int) checkedOut.getChecksum().getValue();
timestamp = System.currentTimeMillis(); // TODO: Do we need to use server synced time?
// Warn: This may become an attack vector! Be careful!
}
file.position(0);
// Write metadata
ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE);
buff.putInt(METADATA_MAGIC_BYTES);
buff.putInt(pos.sectionX);
buff.putInt(Integer.MIN_VALUE); // Unused
buff.putInt(pos.sectionZ);
buff.putInt(checksum);
buff.put(pos.sectionDetail);
buff.put(dataLevel);
buff.put(loaderVersion);
buff.put(Byte.MIN_VALUE); // Unused
buff.putLong(loader.datatypeId);
buff.putLong(timestamp);
LodUtil.assertTrue(buff.remaining() == 0);
buff.flip();
file.write(buff);
}
file.close();
// Atomic move / replace the actual file
Files.move(tempFile.toPath(), path.toPath(), StandardCopyOption.REPLACE_EXISTING,
StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.COPY_ATTRIBUTES);
} finally {
try {
boolean i = tempFile.delete(); // Delete temp file. Ignore errors if fails.
} catch (Exception ignored) {}
}
}
}
src/main/java/com/seibel/lod/core/objects/a7/io/file/DataMetaFile.java
+231
View File
@@ -0,0 +1,231 @@
package com.seibel.lod.core.objects.a7.io.file;
import java.io.*;
import java.lang.ref.SoftReference;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BiConsumer;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.column.DataSourceSaver;
import com.seibel.lod.core.objects.a7.datatype.column.OldDataSourceLoader;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
public class DataMetaFile extends MetaFile {
public static Logger LOGGER = DhLoggerBuilder.getLogger("FileMetadata");
private final DHLevel level;
AtomicInteger localVersion = new AtomicInteger(); // This MUST be atomic
// The '?' type should either be:
// Reference<LodDataSource>, or - Dirtied file that needs to be saved
// SoftReference<LodDataSource>, or - Non-dirty file that can be GCed
// CompletableFuture<LodDataSource>, or - File that is being loaded
// null - Nothing is loaded or being loaded
AtomicReference<Object> data = new AtomicReference<Object>(null);
//TODO: use ConcurrentAppendSingleSwapContainer<LodDataSource> instead of below:
private static class GuardedMultiAppendQueue {
ReentrantReadWriteLock appendLock = new ReentrantReadWriteLock();
ConcurrentLinkedQueue<FullDatatype> queue = new ConcurrentLinkedQueue<>();
}
AtomicReference<GuardedMultiAppendQueue> writeQueue =
new AtomicReference<>(new GuardedMultiAppendQueue());
GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue();
public void addToWriteQueue(FullDatatype datatype) {
GuardedMultiAppendQueue queue = writeQueue.get();
// Using read lock is OK, because the queue's underlying data structure is thread-safe.
// This lock is only used to insure on polling the queue, that the queue is not being
// modified by another thread.
Lock appendLock = queue.appendLock.readLock();
appendLock.lock();
try {
queue.queue.add(datatype);
} finally {
appendLock.unlock();
}
}
private void swapWriteQueue() {
GuardedMultiAppendQueue queue = writeQueue.getAndSet(_backQueue);
// Acquire write lock and then release it again as we only need to ensure that the queue
// is not being appended to by another thread. Note that the above atomic swap &
// the guarantee that all append first acquire the appendLock means after the locK() call,
// there will be no other threads able to or is currently appending to the queue.
// Note: The above needs the getAndSet() to have at least Release Memory order.
// (not that java supports anything non volatile for getAndSet()...)
queue.appendLock.writeLock().lock();
queue.appendLock.writeLock().unlock();
_backQueue = queue;
}
// Load a metaFile in this path. It also automatically read the metadata.
public DataMetaFile(DHLevel level, File path) throws IOException {
super(path);
this.level = level;
}
// Make a new MetaFile. It doesn't load or write any metadata itself.
public DataMetaFile(DHLevel level, File path, DhSectionPos pos) {
super(path, pos);
this.level = level;
}
public boolean isValid(int version) {
boolean isValid = false;
// First check if write queue is empty, then check if localVersion is equal to version.
// Must be done in this order as writer will increment localVersion before polling in the write queue.
// Note: Be careful with the localVerion read's memory order if we do switch over to java 1.9.
// It should be acquire or higher!
isValid = writeQueue.get().queue.isEmpty(); // The 'get()' & 'isEmpty()' enforce a memory barrier.
// Also, we are just querying the state, and this means no
// need to get any locks for the queue.
isValid &= localVersion.get() == version; // The 'get()' enforce a memory barrier.
return isValid;
}
private CompletableFuture<LodDataSource> _readCached(Object obj) {
// Has file cached in RAM and not freed yet.
if ((obj instanceof SoftReference<?>)) {
Object inner = ((SoftReference<?>)obj).get();
if (inner != null) {
LodUtil.assertTrue(inner instanceof LodDataSource);
return CompletableFuture.completedFuture((LodDataSource)inner);
}
}
//==== Cached file out of scrope. ====
// Someone is already trying to complete it. so just return the obj.
if ((obj instanceof CompletableFuture<?>)) {
return (CompletableFuture<LodDataSource>)obj;
}
return null;
}
// Cause: Generic Type runtime casting cannot safety check it.
// However, the Union type ensures the 'data' should only contain the listed type.
public CompletableFuture<LodDataSource> loadOrGetCached(Executor fileReaderThreads) {
Object obj = data.get();
CompletableFuture<LodDataSource> cached = _readCached(obj);
if (cached != null) return cached;
CompletableFuture<LodDataSource> future = new CompletableFuture<LodDataSource>();
// Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :(
boolean worked = data.compareAndSet(obj, future);
if (!worked) return loadOrGetCached(fileReaderThreads);
// Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :(
//return future.completeAsync(this::loadFile, fileReaderThreads);
CompletableFuture.supplyAsync(this::loadAndUpdateDataSource, fileReaderThreads).whenComplete((f, e) -> {
if (e != null) {
LOGGER.error("Uncaught error loading file {}: ", path, e);
future.complete(null);
}
future.complete(f);
});
return future;
}
private LodDataSource loadAndUpdateDataSource() {
LodDataSource data = loadFile();
if (data == null) return null;
// Poll the write queue
// First check if write queue is empty, then swap the write queue.
// Must be done in this order to ensure isValid work properly. See isValid() for details.
boolean isEmpty = writeQueue.get().queue.isEmpty();
int localVer;
if (!isEmpty) {
localVer = localVersion.incrementAndGet();
swapWriteQueue();
// TODO: Use _backQueue to apply the changes into the data.
write(data);
} else localVer = localVersion.get();
data.setLocalVersion(localVer);
// Finally, return the data.
return null;
}
private LodDataSource loadFile() {
// Refresh the metadata.
try {
super.updateMetaData();
} catch (IOException e) {
LOGGER.warn("Metadata for file {} changed unexpectedly and in an invalid state. Dropping file.", path, e);
return null;
}
// Load the file.
try (FileInputStream fio = getDataContent()){
return loader.loadData(this, fio, level);
} catch (IOException e) {
LOGGER.warn("Failed to load file {}. Dropping file.", path, e);
return null;
}
}
private FileInputStream getDataContent() throws IOException {
FileInputStream fin = new FileInputStream(path);
int toSkip = METADATA_SIZE;
while (toSkip > 0) {
long skipped = fin.skip(toSkip);
if (skipped == 0) {
throw new IOException("Invalid file: Failed to skip metadata.");
}
toSkip -= skipped;
}
if (toSkip != 0) {
throw new IOException("File IO Error: Failed to skip metadata.");
}
return fin;
}
public CompletableFuture<Void> flushAndSave(Executor fileWriterThreads) {
boolean isEmpty = writeQueue.get().queue.isEmpty();
if (!isEmpty) {
return loadOrGetCached(fileWriterThreads).thenApply((unused) -> null); // This will flush the data to disk.
} else {
return CompletableFuture.completedFuture(null);
}
}
private void write(LodDataSource data) {
DataSourceSaver saver;
if (loader instanceof DataSourceSaver) saver = (DataSourceSaver) loader;
else if (loader instanceof OldDataSourceLoader) saver = ((OldDataSourceLoader) loader).getNewSaver();
else saver = null;
if (saver == null) return;
BiConsumer<MetaFile, OutputStream> dataWriter = (meta, out) -> {
meta.dataLevel = data.getDataDetail();
meta.dataType = DataSourceLoader.datatypeIdRegistry.get(saver.datatypeId);
meta.loader = saver;
meta.loaderVersion = saver.getSaverVersion();
try {
saver.saveData(level, data, this, out);
} catch (IOException e) {
LOGGER.error("Failed to save data for file {}", path, e);
}
};
try {
super.writeData(dataWriter);
} catch (IOException e) {
LOGGER.error("Failed to write data for file {}", path, e);
}
}
}
src/main/java/com/seibel/lod/core/objects/a7/io/file/LocalDataFileHandler.java
+148
View File
@@ -0,0 +1,148 @@
package com.seibel.lod.core.objects.a7.io.file;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.io.DataSourceProvider;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
public class LocalDataFileHandler implements DataSourceProvider {
// Note: Single main thread only for now. May make it multi-thread later, depending on the usage.
ExecutorService fileReaderThread = LodUtil.makeSingleThreadPool("FileReaderThread");
Logger logger = DhLoggerBuilder.getLogger("LocalDataFileHandler");
ConcurrentHashMap<DhSectionPos, DataMetaFile> files = new ConcurrentHashMap<>();
boolean isScanned = false;
File saveDir;
final DHLevel level;
public LocalDataFileHandler(DHLevel level, File saveRootDir) {
this.saveDir = saveRootDir;
this.level = level;
}
/*
* Caller must ensure that this method is called only once,
* and that this object is not used before this method is called.
*/
public void addScannedFile(Collection<File> detectedFiles) {
HashMultimap<DhSectionPos, DataMetaFile> filesByPos = HashMultimap.create();
{ // Sort files by pos.
for (File file : detectedFiles) {
try {
DataMetaFile metaFile = new DataMetaFile(level, file);
filesByPos.put(metaFile.pos, metaFile);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
// Warn for multiple files with the same pos, and then select the one with latest timestamp.
for (DhSectionPos pos : filesByPos.keySet()) {
Collection<DataMetaFile> metaFiles = filesByPos.get(pos);
DataMetaFile fileToUse;
if (metaFiles.size() > 1) {
fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.timestamp));
{
StringBuilder sb = new StringBuilder();
sb.append("Multiple files with the same pos: ");
sb.append(pos);
sb.append("\n");
for (DataMetaFile metaFile : metaFiles) {
sb.append("\t");
sb.append(metaFile.path);
sb.append("\n");
}
sb.append("\tUsing: ");
sb.append(fileToUse.path);
sb.append("\n");
sb.append("(Other files will be renamed by appending \".old\" to their name.)");
logger.warn(sb.toString());
// Rename all other files with the same pos to .old
for (DataMetaFile metaFile : metaFiles) {
if (metaFile == fileToUse) continue;
File oldFile = new File(metaFile.path + ".old");
try {
if (!metaFile.path.renameTo(oldFile)) throw new RuntimeException("Renaming failed");
} catch (Exception e) {
logger.error("Failed to rename file: " + metaFile.path + " to " + oldFile, e);
}
}
}
} else {
fileToUse = metaFiles.iterator().next();
}
// Add file to the list of files.
files.put(pos, fileToUse);
}
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
@Override
public CompletableFuture<LodDataSource> read(DhSectionPos pos) {
DataMetaFile metaFile = files.get(pos);
if (metaFile == null) {
return CompletableFuture.completedFuture(null);
}
return metaFile.loadOrGetCached(fileReaderThread);
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
@Override
public void write(DhSectionPos sectionPos, FullDatatype chunkData) {
DataMetaFile metaFile = files.get(sectionPos);
if (metaFile != null) { // Fast path: if there is a file for this section, just write to it.
metaFile.addToWriteQueue(chunkData);
return;
}
// Slow path: if there is no file for this section, create one.
DataMetaFile newMetaFile = new DataMetaFile(level, saveDir, sectionPos);
// We add to the queue first so on CAS onto the map, no other thread
// will see the new file without our write entry.
newMetaFile.addToWriteQueue(chunkData);
DataMetaFile casResult = files.putIfAbsent(sectionPos, newMetaFile); // This is a CAS with expected null value.
if (casResult != null) { // another thread already created the file. CAS failed.
// Drop our version and use the cas result.
casResult.addToWriteQueue(chunkData);
}
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
@Override
public CompletableFuture<Void> flushAndSave() {
ArrayList<CompletableFuture<Void>> futures = new ArrayList<CompletableFuture<Void>>();
for (DataMetaFile metaFile : files.values()) {
futures.add(metaFile.flushAndSave(fileReaderThread));
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
private File computeDefaultFilePath(DhSectionPos pos) { //TODO: Temp code as we haven't decided on the file naming & location yet.
return new File(saveDir, pos.serialize() + ".lod");
}
}
src/main/java/com/seibel/lod/core/objects/a7/io/render/RenderFileHandler.java
+43
View File
@@ -0,0 +1,43 @@
package com.seibel.lod.core.objects.a7.io.render;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.objects.a7.RenderDataProvider;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.io.DataSourceProvider;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import com.seibel.lod.core.objects.a7.render.RenderDataSourceLoader;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
public class RenderFileHandler implements RenderDataProvider {
final File renderCacheFolder;
final DataSourceProvider dataSourceProvider;
ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread");
Logger logger = DhLoggerBuilder.getLogger("RenderCache");
public RenderFileHandler(DataSourceProvider sourceProvider, File renderCacheFolder) {
this.dataSourceProvider = sourceProvider;
this.renderCacheFolder = renderCacheFolder;
}
@Override
public CompletableFuture<RenderDataSource> createRenderData(RenderDataSourceLoader renderSourceLoader, DhSectionPos pos) {
return null;
}
public CompletableFuture<RenderDataSource> read(DhSectionPos pos) {
return null;
}
public void write(DhSectionPos sectionPos, FullDatatype chunkData) {
}
}
src/main/java/com/seibel/lod/core/objects/a7/io/render/RenderMetaFile.java
+18
View File
@@ -0,0 +1,18 @@
package com.seibel.lod.core.objects.a7.io.render;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.File;
import java.io.IOException;
public class RenderMetaFile extends MetaFile {
protected RenderMetaFile(File path) throws IOException {
super(path);
}
protected RenderMetaFile(File path, DhSectionPos pos) {
super(path, pos);
}
}
src/main/java/com/seibel/lod/core/objects/a7/pos/DhSectionPos.java
+1 -1
View File
@@ -96,7 +96,7 @@ public class DhSectionPos {
sectionZ == that.sectionZ;
}
// Serialize() is different from toString() as this reqires it to NEVER be changed, and should be in a short format
// Serialize() is different from toString() as this requires it to NEVER be changed, and should be in a short format
public String serialize() {
return "[" + sectionDetail + ',' + sectionX + ',' + sectionZ + ']';
}
src/main/resources/assets/lod/lang/en_us.json
+2 -2
View File
@@ -147,7 +147,7 @@
"lod.config.client.graphics.advancedGraphics.overdrawOffset":
"Overdraw Offset",
"lod.config.client.graphics.advancedGraphics.overdrawOffset.@tooltip":
"If on Vanilla Overdraw mode of NEVER, how much should should the border be offset? \n\n '1': The start of lods will be shifted inwards by 1 chunk, causing 1 chunk of overdraw. \n'-1': The start fo lods will be shifted outwards by 1 chunk, causing 1 chunk of gap. \n\nThis setting can be used to deal with gaps due to our vanilla rendered chunk \n detection not being perfect. \n",
"If Vanilla Overdraw is set to NEVER, how much should the border be offset? \n\n '1': The start of lods will be shifted inwards by 1 chunk, causing 1 chunk of overdraw. \n'-1': The start fo lods will be shifted outwards by 1 chunk, causing 1 chunk of gap. \n\nThis setting can be used to deal with gaps due to our vanilla rendered chunk \n detection not being perfect. \n",
"lod.config.client.graphics.advancedGraphics.useExtendedNearClipPlane":
"Use extended near clip plane",
"lod.config.client.graphics.advancedGraphics.useExtendedNearClipPlane.@tooltip":
@@ -217,7 +217,7 @@
"lod.config.client.multiplayer.multiDimensionRequiredSimilarity.@tooltip":
"When matching worlds of the same dimension type the\ntested chunks must be at least this percent the same\nin order to be considered the same world.\n\nNote: If you use portals to enter a dimension at two\ndifferent locations this system may think it is two different worlds.\n\n§61.0:§r the chunks must be identical.\n§60.5:§r the chunks must be half the same.\n§60.0:§r disables multi-dimension support\n only one world will be used per dimension.",
"lod.config.client.advanced":
"Advance options",
"Advanced options",
"lod.config.client.advanced.threading":
"Threading",
"lod.config.client.advanced.threading.numberOfWorldGenerationThreads":