wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Continue chipping on it

Browse Source
This commit is contained in:
TomTheFurry
2022-06-16 14:23:10 +08:00
parent 742f8b53bb
commit 9ffc6e5a93
9 changed files with 175 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/seibel/lod/core/objects/a7/data/DataFile.java
-179
View File
@@ -1,179 +0,0 @@
package com.seibel.lod.core.objects.a7.data;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.datatype.column.DataSourceSaver;
import com.seibel.lod.core.objects.a7.datatype.column.OldDataSourceLoader;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import java.io.*;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
@Deprecated
public class DataFile {
//Metadata format:
//
// 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signal the metadata format)
// 4 bytes: section X position
// 4 bytes: section Y position (Unused, for future proofing)
// 4 bytes: section Z position
//
// 4 bytes: data checksum //TODO: Implement checksum
// 1 byte: section detail level
// 1 byte: data detail level // Note: not sure if this is needed
// 1 byte: loader version
// 1 byte: unused
//
// 8 bytes: datatype identifier
//
// 8 bytes: unused
// Total size: 32 bytes
public static final int METADATA_SIZE = 32;
public static final int METADATA_MAGIC_BYTES = 0x44_48_76_30;
public final File path;
public final DhSectionPos pos;
public byte dataLevel;
public DataSourceLoader loader;
public byte loaderVersion;
public Class<?> dataType;
public LodDataSource loadedData = null;
public static DataFile readMeta(File path) throws IOException {
try (FileInputStream fin = new FileInputStream(path)) {
MappedByteBuffer buffer = fin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, METADATA_SIZE);
return new DataFile(path, buffer);
}
}
public DataFile(File path, DataSourceLoader loader, LodDataSource loadedData) {
this.path = path;
this.pos = loadedData.getSectionPos();
this.loader = loader;
this.dataType = loader.clazz;
this.dataLevel = loadedData.getDataDetail();
this.loadedData = loadedData;
this.loaderVersion = loader.loaderSupportedVersions[loader.loaderSupportedVersions.length - 1]; // get latest version
}
DataFile(File path, MappedByteBuffer meta) throws IOException {
this.path = path;
int magic = meta.getInt();
if (magic != METADATA_MAGIC_BYTES) {
throw new IOException("Invalid file: Magic bytes check failed.");
}
int x = meta.getInt();
int y = meta.getInt(); // Unused
int z = meta.getInt();
int checksum = meta.getInt();
byte detailLevel = meta.get();
dataLevel = meta.get();
byte loaderVersion = meta.get();
byte unused = meta.get();
long dataTypeId = meta.getLong();
long unused2 = meta.getLong();
LodUtil.assertTrue(meta.remaining() == 0);
this.pos = new DhSectionPos(detailLevel, x, z);
this.loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion);
if (loader == null) {
throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
}
this.dataType = loader.clazz;
this.loaderVersion = loaderVersion;
}
public FileInputStream getDataContent() throws IOException {
FileInputStream fin = new FileInputStream(path);
int toSkip = METADATA_SIZE;
while (toSkip > 0) {
long skipped = fin.skip(toSkip);
if (skipped == 0) {
throw new IOException("Invalid file: Failed to skip metadata.");
}
toSkip -= skipped;
}
if (toSkip != 0) {
throw new IOException("File IO Error: Failed to skip metadata.");
}
return fin;
}
LodDataSource load(DHLevel level) {
throw new UnsupportedOperationException("Deprecated");
}
public boolean verifyPath() {
return path.exists() && path.isFile() && path.canRead() && path.canWrite();
}
public void saveIfNeeded(DHLevel level, boolean freeMemory) {
if (loadedData == null) return;
if (!verifyPath()) return;
try {
save(level, freeMemory);
} catch (IOException e) {
//FIXME: Log and review this handling
}
}
public void save(DHLevel level, boolean freeMemory) throws IOException {
if (loadedData == null) throw new IllegalStateException("No data loaded");
if (!verifyPath()) throw new IOException("File path became invalid");
DataSourceSaver saver;
if (loader instanceof DataSourceSaver) saver = (DataSourceSaver) loader;
else if (loader instanceof OldDataSourceLoader) saver = ((OldDataSourceLoader) loader).getNewSaver();
else saver = null;
if (saver == null) return;
byte newDataLevel = loadedData.getDataDetail();
try (FileOutputStream fout = new FileOutputStream(path, false)) {
try (DataOutputStream out = new DataOutputStream(fout)) {
out.writeInt(METADATA_MAGIC_BYTES);
// Write x, y, z, checksum
out.writeInt(pos.sectionX);
out.writeInt(Integer.MIN_VALUE); // not used for now
out.writeInt(pos.sectionZ);
out.writeInt(Integer.MIN_VALUE); // not used for now
// Write detail level, data level, loader version
out.writeByte(pos.sectionDetail);
out.writeByte(loadedData.getDataDetail());
out.writeByte(saver.getSaverVersion());
// Write unused
out.writeByte((byte) 0);
// Write data type id
out.writeLong(saver.datatypeId);
// Write unused
out.writeLong(Long.MIN_VALUE);
// Write data
saver.saveData(level, loadedData, out);
}
}
dataLevel = newDataLevel;
loader = saver;
if (freeMemory) {
loadedData = null;
}
}
public void close(DHLevel level) {
if (loadedData != null) {
saveIfNeeded(level, true);
}
}
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/ColumnDataLoader.java
+5 -2
View File
@@ -5,6 +5,7 @@ import com.seibel.lod.core.enums.config.EVerticalQuality;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFileHandler;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.io.file.DataMetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
@@ -34,9 +35,11 @@ public class ColumnDataLoader extends DataSourceSaver {
}
@Override
public void saveData(DHLevel level, LodDataSource loadedData, DataOutputStream out) throws IOException {
public void saveData(DHLevel level, LodDataSource loadedData, MetaFile file, OutputStream out) throws IOException {
//TODO: Add compressor here
((ColumnDatatype) loadedData).writeData(out);
try (DataOutputStream dos = new DataOutputStream(out)) {
((ColumnDatatype) loadedData).writeData(dos);
}
}
@Override
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/DataSourceSaver.java
+3 -1
View File
@@ -4,18 +4,20 @@ import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFileHandler;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
public abstract class DataSourceSaver extends DataSourceLoader {
public DataSourceSaver(Class<? extends LodDataSource> clazz, long datatypeId, byte[] loaderSupportedVersions) {
super(clazz, datatypeId, loaderSupportedVersions);
}
public abstract void saveData(DHLevel level, LodDataSource loadedData, DataOutputStream out) throws IOException;
public abstract void saveData(DHLevel level, LodDataSource loadedData, MetaFile file, OutputStream dataStream) throws IOException;
// generate the default file path and file name based on various parameters.
// Ensure the file extension is '.lod'!
public File generateFilePathAndName(File levelFolderPath, DHLevel level, DhSectionPos sectionPos) {
src/main/java/com/seibel/lod/core/objects/a7/io/DataSource.java → src/main/java/com/seibel/lod/core/objects/a7/io/DataSourceProvider.java
+1 -2
View File
@@ -6,9 +6,8 @@ import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.util.concurrent.CompletableFuture;
public interface DataSource {
public interface DataSourceProvider {
CompletableFuture<LodDataSource> read(DhSectionPos pos);
void write(DhSectionPos sectionPos, FullDatatype chunkData);
CompletableFuture<Void> flushAndSave();
}
src/main/java/com/seibel/lod/core/objects/a7/io/MetaFile.java
+58 -4
View File
@@ -1,15 +1,23 @@
package com.seibel.lod.core.objects.a7.io;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.function.BiConsumer;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
import com.seibel.lod.core.objects.a7.data.DataFile;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import net.fabricmc.mapping.tree.Mapped;
public abstract class MetaFile {
public class MetaFile {
//Metadata format:
//
// 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signal the metadata format)
@@ -94,8 +102,8 @@ public abstract class MetaFile {
protected void updateMetaData() throws IOException {
validatePath();
try (FileInputStream fin = new FileInputStream(path)) {
MappedByteBuffer buffer = fin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, METADATA_SIZE);
try (FileChannel channel = FileChannel.open(path.toPath(), StandardOpenOption.READ)) {
MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, METADATA_SIZE);
int magic = buffer.getInt();
if (magic != METADATA_MAGIC_BYTES) {
throw new IOException("Invalid file: Magic bytes check failed.");
@@ -124,4 +132,50 @@ public abstract class MetaFile {
this.loaderVersion = loaderVersion;
}
}
protected void writeData(BiConsumer<MetaFile, OutputStream> dataWriter) throws IOException {
validatePath();
File tempFile = File.createTempFile("", "tmp", path.getParentFile());
tempFile.deleteOnExit();
try (FileChannel file = FileChannel.open(tempFile.toPath(),
StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
{
file.position(METADATA_SIZE);
int checksum;
try (OutputStream channelOut = Channels.newOutputStream(file);
BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer?
CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) { // TODO: Is Adler32 ok?
dataWriter.accept(this, checkedOut);
checksum = (int) checkedOut.getChecksum().getValue();
timestamp = System.currentTimeMillis(); // TODO: Do we need to use server synced time?
// Warn: This may become an attack vector! Be careful!
}
file.position(0);
// Write metadata
ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE);
buff.putInt(METADATA_MAGIC_BYTES);
buff.putInt(pos.sectionX);
buff.putInt(Integer.MIN_VALUE); // Unused
buff.putInt(pos.sectionZ);
buff.putInt(checksum);
buff.put(pos.sectionDetail);
buff.put(dataLevel);
buff.put(loaderVersion);
buff.put(Byte.MIN_VALUE); // Unused
buff.putLong(loader.datatypeId);
buff.putLong(timestamp);
LodUtil.assertTrue(buff.remaining() == 0);
buff.flip();
file.write(buff);
}
file.close();
// Atomic move / replace the actual file
Files.move(tempFile.toPath(), path.toPath(), StandardCopyOption.REPLACE_EXISTING,
StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.COPY_ATTRIBUTES);
} finally {
try {
boolean i = tempFile.delete(); // Delete temp file. Ignore errors if fails.
} catch (Exception ignored) {}
}
}
}
src/main/java/com/seibel/lod/core/objects/a7/io/file/DataMetaFile.java
+39 -8
View File
@@ -1,9 +1,6 @@
package com.seibel.lod.core.objects.a7.io.file;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.*;
import java.lang.ref.SoftReference;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -12,9 +9,14 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BiConsumer;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.column.DataSourceSaver;
import com.seibel.lod.core.objects.a7.datatype.column.OldDataSourceLoader;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
@@ -23,6 +25,8 @@ import org.apache.logging.log4j.Logger;
public class DataMetaFile extends MetaFile {
public static Logger LOGGER = DhLoggerBuilder.getLogger("FileMetadata");
private final DHLevel level;
AtomicInteger localVersion = new AtomicInteger(); // This MUST be atomic
// The '?' type should either be:
@@ -68,13 +72,15 @@ public class DataMetaFile extends MetaFile {
}
// Load a metaFile in this path. It also automatically read the metadata.
public DataMetaFile(File path) throws IOException {
public DataMetaFile(DHLevel level, File path) throws IOException {
super(path);
this.level = level;
}
// Make a new MetaFile. It doesn't load or write any metadata itself.
public DataMetaFile(File path, DhSectionPos pos) {
public DataMetaFile(DHLevel level, File path, DhSectionPos pos) {
super(path, pos);
this.level = level;
}
public boolean isValid(int version) {
@@ -148,7 +154,7 @@ public class DataMetaFile extends MetaFile {
localVer = localVersion.incrementAndGet();
swapWriteQueue();
// TODO: Use _backQueue to apply the changes into the data.
// TODO: Trigger a save to disk.
write(data);
} else localVer = localVersion.get();
data.setLocalVersion(localVer);
// Finally, return the data.
@@ -166,7 +172,7 @@ public class DataMetaFile extends MetaFile {
// Load the file.
try (FileInputStream fio = getDataContent()){
return loader.loadData(this, fio, null); // FIXME: somehow get the level object????
return loader.loadData(this, fio, level);
} catch (IOException e) {
LOGGER.warn("Failed to load file {}. Dropping file.", path, e);
return null;
@@ -197,4 +203,29 @@ public class DataMetaFile extends MetaFile {
return CompletableFuture.completedFuture(null);
}
}
private void write(LodDataSource data) {
DataSourceSaver saver;
if (loader instanceof DataSourceSaver) saver = (DataSourceSaver) loader;
else if (loader instanceof OldDataSourceLoader) saver = ((OldDataSourceLoader) loader).getNewSaver();
else saver = null;
if (saver == null) return;
BiConsumer<MetaFile, OutputStream> dataWriter = (meta, out) -> {
meta.dataLevel = data.getDataDetail();
meta.dataType = DataSourceLoader.datatypeIdRegistry.get(saver.datatypeId);
meta.loader = saver;
meta.loaderVersion = saver.getSaverVersion();
try {
saver.saveData(level, data, this, out);
} catch (IOException e) {
LOGGER.error("Failed to save data for file {}", path, e);
}
};
try {
super.writeData(dataWriter);
} catch (IOException e) {
LOGGER.error("Failed to write data for file {}", path, e);
}
}
}
src/main/java/com/seibel/lod/core/objects/a7/io/file/LocalDataFileHandler.java
+8 -5
View File
@@ -2,9 +2,10 @@ package com.seibel.lod.core.objects.a7.io.file;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.io.DataSource;
import com.seibel.lod.core.objects.a7.io.DataSourceProvider;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
@@ -19,7 +20,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
public class LocalDataFileHandler implements DataSource {
public class LocalDataFileHandler implements DataSourceProvider {
// Note: Single main thread only for now. May make it multi-thread later, depending on the usage.
ExecutorService fileReaderThread = LodUtil.makeSingleThreadPool("FileReaderThread");
Logger logger = DhLoggerBuilder.getLogger("LocalDataFileHandler");
@@ -29,8 +30,10 @@ public class LocalDataFileHandler implements DataSource {
boolean isScanned = false;
File saveDir;
public LocalDataFileHandler(File saveRootDir) {
final DHLevel level;
public LocalDataFileHandler(DHLevel level, File saveRootDir) {
this.saveDir = saveRootDir;
this.level = level;
}
/*
@@ -42,7 +45,7 @@ public class LocalDataFileHandler implements DataSource {
{ // Sort files by pos.
for (File file : detectedFiles) {
try {
DataMetaFile metaFile = new DataMetaFile(file);
DataMetaFile metaFile = new DataMetaFile(level, file);
filesByPos.put(metaFile.pos, metaFile);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -115,7 +118,7 @@ public class LocalDataFileHandler implements DataSource {
}
// Slow path: if there is no file for this section, create one.
DataMetaFile newMetaFile = new DataMetaFile(saveDir, sectionPos);
DataMetaFile newMetaFile = new DataMetaFile(level, saveDir, sectionPos);
// We add to the queue first so on CAS onto the map, no other thread
// will see the new file without our write entry.
src/main/java/com/seibel/lod/core/objects/a7/io/render/RenderFileHandler.java
+43
View File
@@ -0,0 +1,43 @@
package com.seibel.lod.core.objects.a7.io.render;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.objects.a7.RenderDataProvider;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.io.DataSourceProvider;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import com.seibel.lod.core.objects.a7.render.RenderDataSourceLoader;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
public class RenderFileHandler implements RenderDataProvider {
final File renderCacheFolder;
final DataSourceProvider dataSourceProvider;
ExecutorService renderCacheThread = LodUtil.makeSingleThreadPool("RenderCacheThread");
Logger logger = DhLoggerBuilder.getLogger("RenderCache");
public RenderFileHandler(DataSourceProvider sourceProvider, File renderCacheFolder) {
this.dataSourceProvider = sourceProvider;
this.renderCacheFolder = renderCacheFolder;
}
@Override
public CompletableFuture<RenderDataSource> createRenderData(RenderDataSourceLoader renderSourceLoader, DhSectionPos pos) {
return null;
}
public CompletableFuture<RenderDataSource> read(DhSectionPos pos) {
return null;
}
public void write(DhSectionPos sectionPos, FullDatatype chunkData) {
}
}
src/main/java/com/seibel/lod/core/objects/a7/io/render/RenderMetaFile.java
+18
View File
@@ -0,0 +1,18 @@
package com.seibel.lod.core.objects.a7.io.render;
import com.seibel.lod.core.objects.a7.io.MetaFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.File;
import java.io.IOException;
public class RenderMetaFile extends MetaFile {
protected RenderMetaFile(File path) throws IOException {
super(path);
}
protected RenderMetaFile(File path, DhSectionPos pos) {
super(path, pos);
}
}