The very start of file management.

This commit is contained in:
TomTheFurry
2022-05-15 14:38:50 +08:00
parent ad1e3ef62a
commit e0280cc038
8 changed files with 353 additions and 115 deletions
@@ -336,7 +336,7 @@ public abstract class LodQuadTree {
LodSection parent = parentRingList.get(pos.x >> 1, pos.y >> 1);
if (parent == null) {
parent = parentRingList.setChained(pos.x >> 1, pos.y >> 1,
new LodSection(section.pos.getParent(), getRenderDataProvider(), containerType));
new LodSection(section.pos.getParent()));
parent.childCount++;
}
LodUtil.assertTrue(parent.childCount <= 4 && parent.childCount > 0);
@@ -345,7 +345,7 @@ public abstract class LodQuadTree {
LodSection child = childRingList.get(childPos.sectionX, childPos.sectionZ);
if (child == null) {
child = childRingList.setChained(childPos.sectionX, childPos.sectionZ,
new LodSection(childPos, getRenderDataProvider(), containerType));
new LodSection(childPos));
child.childCount = 0;
} else if (child.childCount == -1) {
child.childCount = 0;
@@ -362,7 +362,7 @@ public abstract class LodQuadTree {
}
if (targetLevel <= getDataDetail(f_sectLevel) && section == null) {
section = ringList.setChained(pos.x, pos.y,
new LodSection(sectPos, getRenderDataProvider(), containerType));
new LodSection(sectPos));
}
} else {
// Section is not the top level. So we also need to consider the parent.
@@ -376,12 +376,12 @@ public abstract class LodQuadTree {
}
if (targetLevel < getDataDetail((byte) (f_sectLevel+1)) && section == null) {
section = ringList.setChained(pos.x, pos.y,
new LodSection(sectPos, getRenderDataProvider(), containerType));
new LodSection(sectPos));
LodUtil.assertTrue(parentRingList != null);
LodSection parent = parentRingList.get(pos.x >> 1, pos.y >> 1);
if (parent == null) {
parent = parentRingList.setChained(pos.x >> 1, pos.y >> 1,
new LodSection(sectPos.getParent(), getRenderDataProvider(), containerType));
new LodSection(sectPos.getParent()));
}
parent.childCount++;
}
@@ -424,13 +424,14 @@ public abstract class LodQuadTree {
// Cascade layers
if (doCacsade && section.childCount == 0) {
LodUtil.assertTrue(childRingList != null);
// Create childs to cascade the layer.
for (byte i = 0; i < 4; i++) {
DhSectionPos childPos = section.pos.getChild(i);
LodSection child = childRingList.get(childPos.sectionX, childPos.sectionZ);
if (child == null) {
child = childRingList.setChained(childPos.sectionX, childPos.sectionZ,
new LodSection(childPos, getRenderDataProvider(), containerType));
new LodSection(childPos));
child.childCount = 0;
} else {
LodUtil.assertTrue(child.childCount == -1,
@@ -456,14 +457,17 @@ public abstract class LodQuadTree {
if (section.childCount == -1) LodUtil.assertTrue(
getParentSection(section.pos).childCount == 0);
// Load/unload section
if (section.childCount == 4 && section.isLoaded()) {
section.unload();
} else if (section.childCount == 0 && !section.isLoaded()) {
section.load();
} else if (section.childCount == -1) {
// Call load on new sections, and tick on existing ones, and dispose old sections
if (section.childCount == -1) {
ringList.set(pos.x, pos.y, null);
section.dispose();
} else {
if (!section.isLoaded() && !section.isLoading()) {
section.load(getRenderDataProvider(), containerType);
}
if (section.childCount == 4) section.enableRender();
if (section.childCount == 0) section.disableRender();
section.tick();
}
});
}
@@ -2,7 +2,8 @@ package com.seibel.lod.core.objects.a7;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import com.seibel.lod.core.util.LodUtil;
import java.util.concurrent.CompletableFuture;
public class LodSection {
public static final int SUB_REGION_DATA_WIDTH = 16*16;
@@ -16,39 +17,62 @@ public class LodSection {
// TODO: Should I provide a way to change the render source?
private RenderDataSource renderDataSource;
private boolean isLoaded = false;
private CompletableFuture<RenderDataSource> loadFuture;
private boolean isRenderEnabled = false;
// Create sub region
public LodSection(DhSectionPos pos, RenderDataProvider renderDataProvider, Class<? extends RenderDataSource> renderDataSourceClass) {
public LodSection(DhSectionPos pos) {
this.pos = pos;
this.renderDataSource = renderDataSourceClass == null ?
null : renderDataProvider.createRenderData(pos);
}
public void load() {
public void enableRender() {
if (isRenderEnabled) return;
if (renderDataSource != null) {
LodUtil.assertTrue(!isLoaded());
renderDataSource.load();
isLoaded = true;
renderDataSource.enableRender();
}
isRenderEnabled = true;
}
public void unload() {
public void disableRender() {
if (!isRenderEnabled) return;
if (renderDataSource != null) {
LodUtil.assertTrue(isLoaded());
renderDataSource.unload();
isLoaded = false;
renderDataSource.disableRender();
}
isRenderEnabled = false;
}
public void load(RenderDataProvider renderDataProvider, Class<? extends RenderDataSource> renderDataSourceClass) {
if (loadFuture != null || renderDataSource != null) throw new IllegalStateException("Reloading is not supported!");
loadFuture = renderDataProvider.createRenderData(renderDataSourceClass, pos);
}
public void tick() {
if (loadFuture != null && loadFuture.isDone()) {
renderDataSource = loadFuture.join();
loadFuture = null;
if (isRenderEnabled) {
renderDataSource.enableRender();
}
}
}
public void dispose() {
if (renderDataSource != null) {
if (isLoaded()) renderDataSource.unload();
renderDataSource.dispose();
} else if (loadFuture != null) {
loadFuture.cancel(true);
}
}
public boolean canRender() {
return isLoaded() && renderDataSource.isRenderReady();
}
public boolean isLoaded() {
return renderDataSource != null && isLoaded;
return renderDataSource != null;
}
public boolean isLoading() {
return loadFuture != null;
}
public RenderDataSource getRenderContainer() {
@@ -3,6 +3,8 @@ package com.seibel.lod.core.objects.a7;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import java.util.concurrent.CompletableFuture;
public interface RenderDataProvider {
RenderDataSource createRenderData(DhSectionPos pos);
CompletableFuture<RenderDataSource> createRenderData(RenderDataSource.RenderDataSourceLoader renderSourceLoader, DhSectionPos pos);
}
@@ -0,0 +1,91 @@
package com.seibel.lod.core.objects.a7.data;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
public class DataFile {
//Metadata format:
//
// 4 bytes: magic bytes: "DHv0" (in ascii: 0x44 48 76 30) (this also signal the metadata format)
// 4 bytes: section X position
// 4 bytes: section Y position (Unused, for future proofing)
// 4 bytes: section Z position
//
// 4 bytes: data checksum //TODO: Implement checksum
// 1 byte: section detail level
// 1 byte: data detail level // Note: not sure if this is needed
// 1 byte: loader version
// 1 byte: unused
//
// 8 bytes: datatype identifier
//
// 8 bytes: unused
// Total size: 32 bytes
public static final int METADATA_SIZE = 32;
public static final int METADATA_MAGIC_BYTES = 0x44_48_76_30;
public final File path;
public final DhSectionPos pos;
public final LodDataSource.DataSourceLoader loader;
public final Class<?> dataType;
public LodDataSource loadedData = null;
public static DataFile readMeta(File path) throws IOException {
try (FileInputStream fin = new FileInputStream(path)) {
MappedByteBuffer buffer = fin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, METADATA_SIZE);
return new DataFile(path, buffer);
}
}
public DataFile(File path, DhSectionPos pos, LodDataSource.DataSourceLoader loader, Class<?> dataType) {
this.path = path;
this.pos = pos;
this.loader = loader;
this.dataType = dataType;
}
DataFile(File path, MappedByteBuffer meta) throws IOException {
this.path = path;
int magic = meta.getInt();
if (magic != METADATA_MAGIC_BYTES) {
throw new IOException("Invalid file: Magic bytes check failed.");
}
int x = meta.getInt();
int y = meta.getInt(); // Unused
int z = meta.getInt();
int checksum = meta.getInt();
byte detailLevel = meta.get();
byte dataDetailLevel = meta.get();
byte loaderVersion = meta.get();
byte unused = meta.get();
long dataTypeId = meta.getLong();
long unused2 = meta.getLong();
LodUtil.assertTrue(meta.remaining() == 0);
this.pos = new DhSectionPos(detailLevel, x, z);
this.loader = LodDataSource.getLoader(dataTypeId, loaderVersion);
if (loader == null) {
throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
}
this.dataType = LodDataSource.dataSourceTypeRegistry.get(dataTypeId);
}
LodDataSource load(DHLevel level) throws IOException {
if (loadedData != null) return loadedData;
FileInputStream fin = new FileInputStream(path);
fin.skipNBytes(METADATA_SIZE);
loadedData = loader.loadData(level, pos, fin);
return loadedData;
}
}
@@ -1,5 +1,7 @@
package com.seibel.lod.core.objects.a7.data;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.RenderDataProvider;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.EmptyRenderContainer;
@@ -8,49 +10,80 @@ import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.HashMap;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
public class DataFileHandler implements RenderDataProvider {
public final File folder;
private final HashMap<DhSectionPos, LodDataSource> dataSourceCache;
public static final String FILE_EXTENSION = ".lod";
public DataFileHandler(File folderPath) {
public final DHLevel level;
public final File folder;
private final HashMultimap<DhSectionPos, DataFile> unloadedDataFileCache;
public static final String[] FoldersToScan = {
"data",
}; // TODO: Add more folders to scan
public DataFileHandler(File folderPath, DHLevel level) {
this.folder = folderPath;
dataSourceCache = new HashMap<>();
this.level = level;
unloadedDataFileCache = HashMultimap.create();
File[] foldersToScan = new File[FoldersToScan.length + 1];
for (int i = 0; i < FoldersToScan.length; i++) {
foldersToScan[i] = new File(folder, FoldersToScan[i]);
}
foldersToScan[FoldersToScan.length] = folder;
scanFiles(foldersToScan);
}
public void scanFiles(File[] foldersToScan) {
// Scan all files in the folder and read their metadata
for (File folder : foldersToScan) {
if (!folder.exists() || !folder.isDirectory()) continue;
File[] files = folder.listFiles();
if (files == null) throw new RuntimeException("Could not list files in folder: " + folder.getAbsolutePath());
for (File file : files) {
if (file.isFile()) {
String fileName = file.getName();
if (fileName.endsWith(FILE_EXTENSION)) {
DataFile dataFile;
try {
dataFile = DataFile.readMeta(file);
} catch (IOException e) {
// FIXME: Log error
continue;
}
if (unloadedDataFileCache.containsKey(dataFile.pos)) {
Set<DataFile> fileSet = unloadedDataFileCache.get(dataFile.pos);
if (fileSet.stream().anyMatch(f -> f.dataType.equals(dataFile.dataType))) {
// A file with the same type and same position already exists
// TODO: Handle this case
continue; // For now, ignore the file
}
}
unloadedDataFileCache.put(dataFile.pos, dataFile);
}
}
}
}
}
@Override
public RenderDataSource createRenderData(DhSectionPos pos) {
LodDataSource dataSource = getDataSource(pos);
RenderDataSource renderDataSource = RenderDataSource.tryConstruct(dataSource, pos);
if (renderDataSource == null) renderDataSource = EmptyRenderContainer.INSTANCE;
return renderDataSource;
}
private LodDataSource getDataSource(DhSectionPos pos) {
return dataSourceCache.computeIfAbsent(pos, this::loadOrCreateDataSource);
}
private LodDataSource loadOrCreateDataSource(DhSectionPos pos) {
File dataFile = getDataFile(pos);
if (dataFile.exists()) {
String format = getFormat(dataFile);
try {
LodDataSource data = LodDataSource.loadData(format, new FileInputStream(dataFile));
return data;
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
return new CompleteDataContainer();
}
private String getFormat(File targetFile) {
return null; //TODO
}
private File getDataFile(DhSectionPos pos) {
return null; //TODO
public CompletableFuture<RenderDataSource> createRenderData(RenderDataSource.RenderDataSourceLoader renderSourceLoader, DhSectionPos pos) {
return CompletableFuture.supplyAsync(() -> {
Set<DataFile> files = renderSourceLoader.selectFiles(pos, level, unloadedDataFileCache.get(pos));
LodDataSource[] dataSource = files.stream().map(f -> {
try {
return f.load(level);
} catch (IOException e) {
throw new RuntimeException(e);
}
}).toArray(LodDataSource[]::new);
return renderSourceLoader.construct(dataSource, pos, level);
});
}
}
@@ -1,47 +1,78 @@
package com.seibel.lod.core.objects.a7.data;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Objects;
import java.util.function.Function;
public interface LodDataSource {
String REGISTER_STRING_FILTER_REGEX = "^[a-zA-Z0-9_]*$";
HashMap<String, Function<InputStream,? extends LodDataSource>>
dataSourceLoaderRegistry = new HashMap<String, Function<InputStream,? extends LodDataSource>>();
class LoaderKey {
public final long classId;
public final byte loaderVersion;
public LoaderKey(long classId, byte loaderVersion) {
this.classId = classId;
this.loaderVersion = loaderVersion;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
LoaderKey loaderKey = (LoaderKey) o;
return classId == loaderKey.classId && loaderVersion == loaderKey.loaderVersion;
}
@Override
public int hashCode() {
return Objects.hash(classId, loaderVersion);
}
}
HashMap<LoaderKey, DataSourceLoader>
dataSourceLoaderRegistry = new HashMap<LoaderKey, DataSourceLoader>();
HashMap<Long, Class<?>> dataSourceTypeRegistry = new HashMap<Long, Class<?>>();
interface DataSourceLoader {
// Can return null as meaning the requirement is not met
LodDataSource loadData(DhSectionPos sectionPos, InputStream data);
LodDataSource loadData(DHLevel level, DhSectionPos sectionPos, InputStream data);
}
static void registerDataSourceLoader(String name, int version, Function<InputStream,? extends LodDataSource> loader) {
if (name == null || loader == null || name.isEmpty()) {
throw new IllegalArgumentException("Name and loader must be non-null, and not empty");
static void registerDataSourceLoader(Class<? extends LodDataSource> clazz, long typeId, byte version, DataSourceLoader loader) {
if (loader == null) {
throw new IllegalArgumentException("loader must be non-null");
}
if (!name.matches(REGISTER_STRING_FILTER_REGEX)) {
throw new IllegalArgumentException("Name must pass the regex " + REGISTER_STRING_FILTER_REGEX);
if (dataSourceTypeRegistry.containsKey(typeId) && dataSourceTypeRegistry.get(typeId) != clazz) {
throw new IllegalArgumentException("Loader for typeId " + typeId + " already registered with different class: "
+ dataSourceTypeRegistry.get(typeId) + " != " + clazz);
}
if (dataSourceLoaderRegistry.containsKey(name)) {
throw new IllegalArgumentException("Data source loader already registered for " + name);
LoaderKey key = new LoaderKey(typeId, version);
if (dataSourceLoaderRegistry.containsKey(key)) {
throw new IllegalArgumentException("Data source loader already registered for " + clazz + " with version " + version);
}
dataSourceLoaderRegistry.put(name+"$"+version, loader);
dataSourceLoaderRegistry.put(key, loader);
}
static LodDataSource loadData(String dataSourceTypeNameVersion, InputStream data) {
static DataSourceLoader getLoader(long dataTypeId, byte loaderVersion) {
DataSourceLoader loader = dataSourceLoaderRegistry.get(new LoaderKey(dataTypeId, loaderVersion));
return loader;
}
Function<InputStream,? extends LodDataSource> loader = dataSourceLoaderRegistry.get(dataSourceTypeNameVersion);
static LodDataSource loadData(String dataSourceTypeNameVersion, DHLevel level, DhSectionPos pos, InputStream data) {
DataSourceLoader loader = dataSourceLoaderRegistry.get(dataSourceTypeNameVersion);
if (loader == null) {
throw new IllegalArgumentException("No loader for data source type " + dataSourceTypeNameVersion);
}
return loader.apply(data);
return loader.loadData(level, pos, data);
}
DataSourceLoader getLatestLoader();
<T> T[] getData(); //TODO & FIXME: What is T?
DhSectionPos getSectionPos();
byte getDataDetail();
}
@@ -1,6 +1,7 @@
package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.objects.LodDataView;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
@@ -799,21 +800,40 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
}
}
public static class ColumnRenderSourceLoader extends RenderDataSourceLoader {
@Override
public RenderDataSource construct(LodDataSource[] dataSources, DhSectionPos sectionPos, DHLevel level) {
// Select the direct one first
for (LodDataSource dataSource : dataSources) {
if (dataSource instanceof ColumnDatatype) {
return (RenderDataSource) dataSource;
}
}
// Select the one that is from lower level
}
}
public static RenderDataSource loadByCasting(LodDataSource dataSource, DhSectionPos sectionPos) {
if (dataSource instanceof ColumnDatatype) {
return (RenderDataSource) dataSource;
}
return null;
}
public static RenderDataSource loadByCopying(LodDataSource dataSource, DhSectionPos sectionPos) {
ColumnDatatype columns = new ColumnDatatype(sectionPos, dataSource,
DetailDistanceUtil.getMaxVerticalData(sectionPos.dataDetail));
return null;
}
static {
RenderDataSource.registorLoader(ColumnDatatype::loadByCasting, 100);
}
// public static RenderDataSource loadByCopying(LodDataSource dataSource, DhSectionPos sectionPos) {
//
// ColumnDatatype columns = new ColumnDatatype(sectionPos, dataSource,
// DetailDistanceUtil.getMaxVerticalData(dataDetail));
// //TODO
//
// return null;
// }
// static {
// RenderDataSource.registorLoader(ColumnDatatype::loadByCasting, 100);
// }
@Override
public DataSourceLoader getLatestLoader() {
@@ -831,17 +851,34 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
}
@Override
public void load() {
public byte getDataDetail() {
return (byte) (sectionPos.sectionDetail - SECTION_SIZE_OFFSET);
}
@Override
public void unload() {
public void enableRender() {
}
@Override
public void disableRender() {
}
@Override
public boolean isRenderReady() {
return false;
}
@Override
public void dispose() {
}
@Override
public byte getDetailOffset() {
return 0;
}
@Override
public boolean trySwapRenderBuffer(AtomicReference<RenderBuffer> referenceSlot) {
return false;
@@ -1,10 +1,12 @@
package com.seibel.lod.core.objects.a7.render;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.data.DataFile;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.opengl.RenderBuffer;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -21,32 +23,46 @@ import java.util.concurrent.atomic.AtomicReference;
</pre>
*/
public interface RenderDataSource {
interface RenderContainerConstructor {
// Can return null as meaning the requirement is not met
RenderDataSource testAndConstruct(LodDataSource dataSource, DhSectionPos sectionPos);
}
SortedMap<Integer, RenderContainerConstructor>
renderContainerLoaderRegistry = new TreeMap<Integer, RenderContainerConstructor>();
static void registorLoader(RenderContainerConstructor func, int priority) {
if (func == null) {
throw new IllegalArgumentException("loader must be non-null");
// Don't think this is needed with the newer quad tree structure...
// interface RenderContainerConstructor {
// // Can return null as meaning the requirement is not met
// RenderDataSource testAndConstruct(LodDataSource dataSource, DhSectionPos sectionPos);
// }
// SortedMap<Integer, RenderContainerConstructor>
// renderContainerLoaderRegistry = new TreeMap<Integer, RenderContainerConstructor>();
// static void registorLoader(RenderContainerConstructor func, int priority) {
// if (func == null) {
// throw new IllegalArgumentException("loader must be non-null");
// }
// renderContainerLoaderRegistry.put(priority, func);
// }
//
// static RenderDataSource tryConstruct(LodDataSource dataSource, DhSectionPos pos) {
// for (RenderContainerConstructor func : renderContainerLoaderRegistry.values()) {
// RenderDataSource container = func.testAndConstruct(dataSource, pos);
// if (container != null) {
// return container;
// }
// }
// return null;
// }
abstract class RenderDataSourceLoader {
public abstract RenderDataSource construct(LodDataSource[] dataSources, DhSectionPos sectionPos, DHLevel level);
public Set<DataFile> selectFiles(DhSectionPos sectionPos, DHLevel level, Set<DataFile> availableFiles) {
return Collections.singleton(availableFiles.iterator().next());
}
renderContainerLoaderRegistry.put(priority, func);
}
static RenderDataSource tryConstruct(LodDataSource dataSource, DhSectionPos pos) {
for (RenderContainerConstructor func : renderContainerLoaderRegistry.values()) {
RenderDataSource container = func.testAndConstruct(dataSource, pos);
if (container != null) {
return container;
}
}
return null;
}
void load(); // notify the container that it is now loaded and therefore may be rendered
void unload(); // notify the container that it is now unloaded and therefore will not be rendered
void enableRender();
void disableRender();
boolean isRenderReady();
void dispose(); // notify the container that the parent lodSection is now disposed (can be in loaded or unloaded state)
byte getDetailOffset();
/**
* Try and swap in new render buffer for this section. Note that before this call, there should be no other
* places storing or referencing the render buffer.