wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Continue work on Saving the data.

Browse Source
This commit is contained in:
TomTheFurry
2022-05-21 14:46:17 +08:00
parent af82e4a6d4
commit b391252c03
14 changed files with 598 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/seibel/lod/core/objects/a7/data/CompleteDataContainer.java
-2
View File
@@ -5,10 +5,8 @@ import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.function.Function;
public class CompleteDataContainer implements LodDataSource { // 1 chunk
private DhSectionPos sectionPos;
src/main/java/com/seibel/lod/core/objects/a7/data/DataFile.java
+61 -12
View File
@@ -1,12 +1,12 @@
package com.seibel.lod.core.objects.a7.data;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.datatype.column.DataSourceSaver;
import com.seibel.lod.core.objects.a7.datatype.column.OldDataSourceLoader;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.*;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
@@ -35,9 +35,9 @@ public class DataFile {
public final File path;
public final DhSectionPos pos;
public final byte dataLevel;
public final LodDataSource.DataSourceLoader loader;
public final Class<?> dataType;
public byte dataLevel;
public DataSourceLoader loader;
public Class<?> dataType;
public LodDataSource loadedData = null;
@@ -48,12 +48,13 @@ public class DataFile {
}
}
public DataFile(File path, DhSectionPos pos, LodDataSource.DataSourceLoader loader, Class<?> dataType, byte dataLevel) {
public DataFile(File path, DataSourceLoader loader, LodDataSource loadedData) {
this.path = path;
this.pos = pos;
this.pos = loadedData.getSectionPos();
this.loader = loader;
this.dataType = dataType;
this.dataLevel = dataLevel;
this.dataType = loader.clazz;
this.dataLevel = loadedData.getDataDetail();
this.loadedData = loadedData;
}
DataFile(File path, MappedByteBuffer meta) throws IOException {
@@ -76,11 +77,11 @@ public class DataFile {
LodUtil.assertTrue(meta.remaining() == 0);
this.pos = new DhSectionPos(detailLevel, x, z);
this.loader = LodDataSource.getLoader(dataTypeId, loaderVersion);
this.loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion);
if (loader == null) {
throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
}
this.dataType = LodDataSource.dataSourceTypeRegistry.get(dataTypeId);
this.dataType = loader.clazz;
}
LodDataSource load(DHLevel level) throws IOException {
@@ -90,4 +91,52 @@ public class DataFile {
loadedData = loader.loadData(level, pos, fin);
return loadedData;
}
public boolean verifyPath() {
return path.exists() && path.isFile() && path.canRead() && path.canWrite();
}
public void save(DHLevel level) throws IOException {
if (loadedData == null) throw new IllegalStateException("No data loaded");
if (!verifyPath()) throw new IOException("File path became invalid");
DataSourceSaver saver;
if (loader instanceof DataSourceSaver) saver = (DataSourceSaver) loader;
else if (loader instanceof OldDataSourceLoader) saver = ((OldDataSourceLoader) loader).getNewSaver();
else throw new IllegalStateException("Data source does not support saving");
byte newDataLevel = loadedData.getDataDetail();
try (FileOutputStream fout = new FileOutputStream(path, false)) {
try (DataOutputStream out = new DataOutputStream(fout)) {
out.writeInt(METADATA_MAGIC_BYTES);
// Write x, y, z, checksum
out.writeInt(pos.sectionX);
out.writeInt(Integer.MIN_VALUE); // not used for now
out.writeInt(pos.sectionZ);
out.writeInt(Integer.MIN_VALUE); // not used for now
// Write detail level, data level, loader version
out.writeByte(pos.sectionDetail);
out.writeByte(loadedData.getDataDetail());
out.writeByte(saver.loaderVersion);
// Write unused
out.writeByte((byte) 0);
// Write data type id
out.writeLong(saver.datatypeId);
// Write unused
out.writeLong(Long.MIN_VALUE);
// Write data
saver.saveData(level, loadedData, out);
}
}
dataLevel = newDataLevel;
loader = saver;
}
}
src/main/java/com/seibel/lod/core/objects/a7/data/DataFileHandler.java
+53 -11
View File
@@ -1,9 +1,9 @@
package com.seibel.lod.core.objects.a7.data;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.objects.Pos2D;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.RenderDataProvider;
import com.seibel.lod.core.objects.a7.datatype.column.DataSourceSaver;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import com.seibel.lod.core.objects.a7.render.RenderDataSourceLoader;
@@ -11,11 +11,14 @@ import com.seibel.lod.core.util.LodUtil;
import java.io.File;
import java.io.IOException;
import java.rmi.server.ExportException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
public class DataFileHandler implements RenderDataProvider {
public static final List<OldFileConverter> CONVERTERS = new ArrayList<>();
public static final String FILE_EXTENSION = ".lod";
public final DHLevel level;
@@ -23,6 +26,8 @@ public class DataFileHandler implements RenderDataProvider {
public final File folder;
// A hash map of all data files.
public final ExecutorService IO_MANAGER = LodUtil.makeSingleThreadPool("DataFileHandler IO Manager");
private byte maxDataLevel = 0;
private final HashMultimap<DhSectionPos, DataFile> dataFiles;
@@ -35,7 +40,12 @@ public class DataFileHandler implements RenderDataProvider {
this.folder = folderPath;
this.level = level;
dataFiles = HashMultimap.create();
// Handle converting old files that doesn't have the meta data and stuff
List<DataFile> oldFiles = new ArrayList<>();
for (OldFileConverter converter : CONVERTERS) oldFiles.addAll(converter.scanAndConvert(folder, level));
oldFiles.forEach(this::_addFile);
// Scan for files
File[] foldersToScan = new File[FoldersToScan.length + 1];
for (int i = 0; i < FoldersToScan.length; i++) {
foldersToScan[i] = new File(folder, FoldersToScan[i]);
@@ -54,6 +64,18 @@ public class DataFileHandler implements RenderDataProvider {
}
return files;
}
private void _addFile(DataFile file) {
if (dataFiles.containsKey(file.pos)) {
Set<DataFile> fileSet = dataFiles.get(file.pos);
if (fileSet.stream().anyMatch(f -> f.dataType.equals(file.dataType))) {
// A file with the same type and same position already exists
// TODO: Handle this case
return;
}
}
maxDataLevel = LodUtil.max(maxDataLevel, file.dataLevel);
dataFiles.put(file.pos, file);
}
public void scanFiles(File[] foldersToScan) {
// Scan all files in the folder and read their metadata
@@ -73,22 +95,42 @@ public class DataFileHandler implements RenderDataProvider {
// FIXME: Log error
continue;
}
if (dataFiles.containsKey(dataFile.pos)) {
Set<DataFile> fileSet = dataFiles.get(dataFile.pos);
if (fileSet.stream().anyMatch(f -> f.dataType.equals(dataFile.dataType))) {
// A file with the same type and same position already exists
// TODO: Handle this case
continue; // For now, ignore the file
}
}
maxDataLevel = LodUtil.max(maxDataLevel, dataFile.dataLevel);
dataFiles.put(dataFile.pos, dataFile);
_addFile(dataFile);
}
}
}
}
}
public DataFile registerNewLodDataSource(LodDataSource dataSource, DataSourceSaver saver) {
DhSectionPos pos = dataSource.getSectionPos();
File newFile = saver.generateFilePathAndName(folder, level, pos);
if (!newFile.getName().endsWith(FILE_EXTENSION)) {
//TODO: Log warning
newFile = new File(newFile.getParentFile(), newFile.getName() + FILE_EXTENSION);
}
if (newFile.exists()) {
//TODO: Log warning
String fileStr = newFile.getPath().substring(0, newFile.getPath().length() - FILE_EXTENSION.length());
int i = 1;
do {
newFile = new File(fileStr + "_" + i + FILE_EXTENSION);
i++;
} while (newFile.exists());
}
DataFile dataFile = new DataFile(newFile, saver, dataSource);
dataFiles.put(pos, dataFile);
try {
dataFile.save(level);
} catch (Exception e) {
dataFiles.remove(pos, dataFile);
//TODO: Log error
return null;
}
return dataFile;
}
@Override
public CompletableFuture<RenderDataSource> createRenderData(RenderDataSourceLoader renderSourceLoader, DhSectionPos pos) {
src/main/java/com/seibel/lod/core/objects/a7/data/DataSourceLoader.java
+51
View File
@@ -0,0 +1,51 @@
package com.seibel.lod.core.objects.a7.data;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.DataOutputStream;
import java.io.File;
import java.io.InputStream;
import java.util.*;
public abstract class DataSourceLoader {
public static final HashMultimap<Long, DataSourceLoader> loaderRegistry = HashMultimap.create();
public static final HashMap<Long, Class<? extends LodDataSource>> datatypeIdRegistry = new HashMap<>();
public final long datatypeId;
public final byte loaderVersion;
public final Class<? extends LodDataSource> clazz;
public DataSourceLoader(Class<? extends LodDataSource> clazz, long datatypeId, byte loaderVersion) {
this.datatypeId = datatypeId;
this.loaderVersion = loaderVersion;
this.clazz = clazz;
if (datatypeIdRegistry.containsKey(datatypeId) && datatypeIdRegistry.get(datatypeId) != clazz) {
throw new IllegalArgumentException("Loader for datatypeId " + datatypeId + " already registered with different class: "
+ datatypeIdRegistry.get(datatypeId) + " != " + clazz);
}
Set<DataSourceLoader> loaders = loaderRegistry.get(datatypeId);
if (loaders.stream().anyMatch(l -> l.loaderVersion == loaderVersion)) {
throw new IllegalArgumentException("Loader for class " + clazz + " with version " + loaderVersion " already registered!");
}
datatypeIdRegistry.put(datatypeId, clazz);
loaderRegistry.put(datatypeId, this);
}
// Can return null as meaning the requirement is not met
public abstract LodDataSource loadData(DHLevel level, DhSectionPos sectionPos, InputStream data);
public List<File> foldersToScan(File levelFolderPath) {
return Collections.emptyList();
}
public static DataSourceLoader getLoader(long dataTypeId, byte loaderVersion) {
return loaderRegistry.get(dataTypeId).stream()
.filter(l -> l.loaderVersion == loaderVersion)
.findFirst().orElse(null);
}
}
src/main/java/com/seibel/lod/core/objects/a7/data/LodDataSource.java
-65
View File
@@ -1,78 +1,13 @@
package com.seibel.lod.core.objects.a7.data;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Objects;
import java.util.function.Function;
public interface LodDataSource {
class LoaderKey {
public final long classId;
public final byte loaderVersion;
public LoaderKey(long classId, byte loaderVersion) {
this.classId = classId;
this.loaderVersion = loaderVersion;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
LoaderKey loaderKey = (LoaderKey) o;
return classId == loaderKey.classId && loaderVersion == loaderKey.loaderVersion;
}
@Override
public int hashCode() {
return Objects.hash(classId, loaderVersion);
}
}
HashMap<LoaderKey, DataSourceLoader>
dataSourceLoaderRegistry = new HashMap<LoaderKey, DataSourceLoader>();
HashMap<Long, Class<?>> dataSourceTypeRegistry = new HashMap<Long, Class<?>>();
interface DataSourceLoader {
// Can return null as meaning the requirement is not met
LodDataSource loadData(DHLevel level, DhSectionPos sectionPos, InputStream data);
}
static void registerDataSourceLoader(Class<? extends LodDataSource> clazz, long typeId, byte version, DataSourceLoader loader) {
if (loader == null) {
throw new IllegalArgumentException("loader must be non-null");
}
if (dataSourceTypeRegistry.containsKey(typeId) && dataSourceTypeRegistry.get(typeId) != clazz) {
throw new IllegalArgumentException("Loader for typeId " + typeId + " already registered with different class: "
+ dataSourceTypeRegistry.get(typeId) + " != " + clazz);
}
LoaderKey key = new LoaderKey(typeId, version);
if (dataSourceLoaderRegistry.containsKey(key)) {
throw new IllegalArgumentException("Data source loader already registered for " + clazz + " with version " + version);
}
dataSourceLoaderRegistry.put(key, loader);
}
static DataSourceLoader getLoader(long dataTypeId, byte loaderVersion) {
DataSourceLoader loader = dataSourceLoaderRegistry.get(new LoaderKey(dataTypeId, loaderVersion));
return loader;
}
static LodDataSource loadData(String dataSourceTypeNameVersion, DHLevel level, DhSectionPos pos, InputStream data) {
DataSourceLoader loader = dataSourceLoaderRegistry.get(dataSourceTypeNameVersion);
if (loader == null) {
throw new IllegalArgumentException("No loader for data source type " + dataSourceTypeNameVersion);
}
return loader.loadData(level, pos, data);
}
DataSourceLoader getLatestLoader();
<T> T[] getData(); //TODO & FIXME: What is T?
DhSectionPos getSectionPos();
byte getDataDetail();
}
src/main/java/com/seibel/lod/core/objects/a7/data/OldFileConverter.java
+10
View File
@@ -0,0 +1,10 @@
package com.seibel.lod.core.objects.a7.data;
import com.seibel.lod.core.objects.a7.DHLevel;
import java.io.File;
import java.util.List;
public interface OldFileConverter {
List<DataFile> scanAndConvert(File levelFolder, DHLevel level);
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/Alpha6DataLoader.java
+102
View File
@@ -0,0 +1,102 @@
package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.enums.config.VerticalQuality;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFile;
import com.seibel.lod.core.objects.a7.data.DataFileHandler;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.data.OldFileConverter;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
public class Alpha6DataLoader extends OldDataSourceLoader implements OldFileConverter {
public Alpha6DataLoader() {
super(ColumnDatatype.class, ColumnDatatype.DATA_TYPE_ID, (byte)0);
DataFileHandler.CONVERTERS.add(this);
}
@Override
public LodDataSource loadData(DHLevel level, DhSectionPos sectionPos, InputStream data) {
return null;
}
@Override
public DataSourceSaver getNewSaver() {
return null;
}
private static DataFile convert(File file, int detailLevel, VerticalQuality quality) {
String oldName = file.getName();
String regionStr = oldName.substring("lod.".length(), oldName.length() - ".xz".length());
String[] parts = regionStr.split("\\.");
if (parts.length != 2) return null;
int regionX = Integer.parseInt(parts[0]);
int regionZ = Integer.parseInt(parts[1]);
ColumnDatatype datatype;
try (FileInputStream fileInStream = new FileInputStream(file)) {
XZCompressorInputStream inputStream = new XZCompressorInputStream(fileInStream);
int fileVersion = inputStream.read();
datatype = new ColumnDatatype(fileVersion, quality, detailLevel);
DhSectionPos pos;
File newFilePath = ColumnDataLoader.INSTANCE.generateFilePathAndName(file,
detailLevel,
quality);
} catch (Exception e) {
e.printStackTrace();
}
int version
}
@Override
public List<DataFile> scanAndConvert(File levelFolder, DHLevel level) {
List<DataFile> files = new ArrayList<>();
List<File> foldersToScan = new ArrayList<>(VerticalQuality.values().length);
for (VerticalQuality q : VerticalQuality.values()) {
File qualityFolder = new File(levelFolder, q.toString());
for (int i = 0; i < 10; i++) {
foldersToScan.add(new File(qualityFolder, "detail-"+i));
}
}
for (VerticalQuality q : VerticalQuality.values()) {
for (int i = 0; i < 10; i++) {
File detailFolder = new File(levelFolder, q.toString() + File.pathSeparator + "detail-" + i);
if (!detailFolder.exists() || !detailFolder.isDirectory()) continue;
File[] filesToScan = detailFolder.listFiles();
if (filesToScan == null) continue;
for (File f : filesToScan) {
String fileName = f.getName();
if (!fileName.endsWith(".xz") || fileName.startsWith("lod.")) continue;
DataFile converted = convert(f, i, q);
}
}
}
return files;
}
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/ColumnDataLoader.java
+61
View File
@@ -0,0 +1,61 @@
package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.Config;
import com.seibel.lod.core.enums.config.VerticalQuality;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFileHandler;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.Inherited;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
public class ColumnDataLoader extends DataSourceSaver {
private static final byte COLUMN_DATA_LOADER_VERSION = 10;
public static final ColumnDataLoader INSTANCE = new ColumnDataLoader();
private ColumnDataLoader() {
super(ColumnDatatype.class, ColumnDatatype.DATA_TYPE_ID, COLUMN_DATA_LOADER_VERSION);
}
@Override
public LodDataSource loadData(DHLevel level, DhSectionPos sectionPos, InputStream data) {
//TODO: Add decompressor here
return ColumnDatatype.loadFile(level, sectionPos, data, COLUMN_DATA_LOADER_VERSION);
}
@Override
public void saveData(DHLevel level, LodDataSource loadedData, DataOutputStream out) throws IOException {
//TODO: Add compressor here
((ColumnDatatype) loadedData).writeData(out);
}
@Override
public File generateFilePathAndName(File levelFolderPath, DHLevel level, DhSectionPos sectionPos) {
return generateFilePathAndName(levelFolderPath, sectionPos, Config.Client.Graphics.Quality.verticalQuality.get());
}
public File generateFilePathAndName(File levelFolderPath, DhSectionPos sectionPos, VerticalQuality quality) {
return new File(levelFolderPath, "cache" + File.separator + quality.toString() + File.separator +
String.format("%s_v%d-%s%s", clazz.getSimpleName(), loaderVersion,
sectionPos.serialize(), DataFileHandler.FILE_EXTENSION));
}
@Override
public List<File> foldersToScan(File levelFolderPath) {
File cacheFolder = new File(levelFolderPath, "cache");
List<File> foldersToScan = new ArrayList<>(VerticalQuality.values().length);
for (VerticalQuality q : VerticalQuality.values()) {
foldersToScan.add(new File(cacheFolder, q.toString()));
}
return foldersToScan;
}
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/ColumnDatatype.java
+12 -180
View File
@@ -3,14 +3,12 @@ package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.objects.LodDataView;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.LodQuadTree;
import com.seibel.lod.core.objects.a7.data.DataFile;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import com.seibel.lod.core.objects.a7.render.RenderDataSourceLoader;
import com.seibel.lod.core.objects.opengl.RenderBuffer;
import com.seibel.lod.core.util.DetailDistanceUtil;
import com.seibel.lod.core.util.LodUtil;
import java.io.DataInputStream;
@@ -19,7 +17,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
public class ColumnDatatype implements LodDataSource, RenderDataSource {
@@ -70,6 +67,7 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
dataContainer = readDataVersion8(inputData, verticalSize);
break;
case 9:
case 10:
dataContainer = readDataVersion9(inputData, verticalSize);
break;
default:
@@ -100,6 +98,7 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
fileDataContainer = readDataVersion8(inputData, fileMaxVerticalSize);
break;
case 9:
case 10:
fileDataContainer = readDataVersion9(inputData, fileMaxVerticalSize);
break;
default:
@@ -216,7 +215,6 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
{
return ColumnDataPoint.doesItExist(getSingleData(posX, posZ));
}
private long[] readDataVersion6(DataInputStream inputData, int tempMaxVerticalData) throws IOException {
int x = SECTION_SIZE * SECTION_SIZE * tempMaxVerticalData;
byte[] data = new byte[x * Long.BYTES];
@@ -239,7 +237,6 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
patchHeightAndDepth(result, 64 - yOffset);
return result;
}
private long[] readDataVersion8(DataInputStream inputData, int tempMaxVerticalData) throws IOException {
int x = SECTION_SIZE * SECTION_SIZE * tempMaxVerticalData;
byte[] data = new byte[x * Long.BYTES];
@@ -254,7 +251,6 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
}
return result;
}
private long[] readDataVersion9(DataInputStream inputData, int tempMaxVerticalData) throws IOException {
int x = SECTION_SIZE * SECTION_SIZE * tempMaxVerticalData;
byte[] data = new byte[x * Long.BYTES];
@@ -293,7 +289,7 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
outputView.mergeMultiDataFrom(quadView);
}
public boolean writeData(DataOutputStream output) throws IOException {
boolean writeData(DataOutputStream output) throws IOException {
output.writeByte(getDataDetail());
output.writeByte((byte) verticalSize);
// FIXME: yOffset is a int, but we only are writing a short.
@@ -349,7 +345,9 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
return (long) dataContainer.length * Long.BYTES;
}
public static LodDataSource loadFile(DHLevel level, DhSectionPos pos, InputStream is, int version) {
// Called by ColumnDataLoader
static LodDataSource loadFile(DHLevel level, DhSectionPos pos, InputStream is, int version) {
try (DataInputStream dis = new DataInputStream(is)) {
return new ColumnDatatype(pos, dis, version, level);
} catch (IOException e) {
@@ -358,182 +356,15 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
}
}
public static RenderDataSourceLoader COLUMN_LAYER_LOADER = new RenderDataSourceLoader(4) {
@Override
public RenderDataSource construct(List<LodDataSource> dataSources, DhSectionPos sectionPos, DHLevel level) {
if (dataSources.size() == 0) return null;
// Check for direct casting
if (dataSources.size() == 1 && dataSources.get(0) instanceof ColumnDatatype
&& dataSources.get(0).getSectionPos().equals(sectionPos)
&& dataSources.get(0).getDataDetail() == sectionPos.sectionDetail-SECTION_SIZE_OFFSET) {
// Directly using the data source as the render data source is possible.
return (ColumnDatatype) dataSources.get(0);
}
// Otherwise, we need to create a new render data source, and copy the data from the data sources.
ColumnDatatype renderDataSource = new ColumnDatatype(sectionPos,
DetailDistanceUtil.getMaxVerticalData(sectionPos.sectionDetail-SECTION_SIZE_OFFSET),
level.getMinY());
boolean completeCopy = dataSources.get(0).getSectionPos().getWidth().toBlock() >= sectionPos.getWidth().toBlock();
if (completeCopy) {
// If there is only one data source, we need to insure on copy, we don't copy out of bounds as we
// may just need to copy partial section of the data source.
LodUtil.assertTrue(dataSources.size() == 1, "Expected only one data source for complete copy");
byte targetDataLevel = (byte) (sectionPos.sectionDetail-SECTION_SIZE_OFFSET);
byte sourceDataLevel = dataSources.get(0).getDataDetail();
LodUtil.assertTrue(targetDataLevel >= sourceDataLevel);
if (dataSources.get(0) instanceof ColumnDatatype) {
ColumnDatatype dataSource = (ColumnDatatype) dataSources.get(0);
DhSectionPos srcPos = dataSource.getSectionPos();
// Note that in here, the source data level will be always < target section level
int trgX = sectionPos.getCorner().getX().toBlock();
int trgZ = sectionPos.getCorner().getZ().toBlock();
int trgMaxX = trgX + sectionPos.getWidth().toBlock() - 1;
int trgMaxZ = trgZ + sectionPos.getWidth().toBlock() - 1;
int trgXSizeInSrc = (trgX >> sourceDataLevel) - (trgMaxX >> sourceDataLevel) + 1;
int trgZSizeInSrc = (trgZ >> sourceDataLevel) - (trgMaxZ >> sourceDataLevel) + 1;
int trgXInSrc = (trgX >> sourceDataLevel) % srcPos.getWidth(sourceDataLevel).value;
int trgZInSrc = (trgZ >> sourceDataLevel) % srcPos.getWidth(sourceDataLevel).value;
ColumnQuadView srcView = dataSource.getDataInQuad(trgXInSrc, trgZInSrc, trgXSizeInSrc, trgZSizeInSrc);
ColumnQuadView trgView = renderDataSource.getFullQuad();
trgView.mergeMultiColumnFrom(srcView);
} else {
if (!(dataSources.get(0) instanceof FullDatatype))
throw new IllegalArgumentException("Unsupported data source type: " + dataSources.get(0).getClass().getName());
FullDatatype dataSource = (FullDatatype) dataSources.get(0);
DhSectionPos srcPos = dataSource.getSectionPos();
//TODO: Impl this
LodUtil.assertTrue(false,"Not implemented yet");
}
} else {
// If there are multiple data sources, we need to merge them into the target data source
for (LodDataSource dataSource : dataSources) {
byte targetDataLevel = (byte) (sectionPos.sectionDetail-SECTION_SIZE_OFFSET);
byte sourceDataLevel = dataSource.getDataDetail();
DhSectionPos srcPos = dataSource.getSectionPos();
if (dataSource instanceof ColumnDatatype) {
ColumnDatatype clDataSource = (ColumnDatatype) dataSource;
// Note that targetDataLevel can be > source section level
int srcX = srcPos.getCorner().getX().toBlock();
int srcZ = srcPos.getCorner().getZ().toBlock();
int srcMaxX = srcX + srcPos.getWidth().toBlock() - 1;
int srcMaxZ = srcZ + srcPos.getWidth().toBlock() - 1;
int srcXSizeInTrg = (srcX >> targetDataLevel) - (srcMaxX >> targetDataLevel) + 1;
int srcZSizeInTrg = (srcZ >> targetDataLevel) - (srcMaxZ >> targetDataLevel) + 1;
int srcXInTrg = (srcX >> targetDataLevel) % SECTION_SIZE;
int srcZInTrg = (srcZ >> targetDataLevel) % SECTION_SIZE;
ColumnQuadView srcView = clDataSource.getFullQuad();
ColumnQuadView trgView = renderDataSource.getDataInQuad(srcXInTrg, srcZInTrg, srcXSizeInTrg, srcZSizeInTrg);
trgView.mergeMultiColumnFrom(srcView);
} else {
if (!(dataSource instanceof FullDatatype))
throw new IllegalArgumentException("Unsupported data source type: " + dataSource.getClass().getName());
FullDatatype flDataSource = (FullDatatype) dataSource;
//TODO: Impl this
LodUtil.assertTrue(false,"Not implemented yet");
}
}
}
return renderDataSource;
}
@Override
public List<DataFile> selectFiles(DhSectionPos sectionPos, DHLevel level, List<DataFile>[] availableFiles) {
byte targetDataLevel = (byte) (sectionPos.sectionDetail - SECTION_SIZE_OFFSET);
//No support for loading higher than the target level yet.
byte maxDataLevel = LodUtil.min((byte) (availableFiles.length-1), targetDataLevel);
byte topValidDataLevel = Byte.MIN_VALUE;
List<DataFile> selectedFiles = new LinkedList<>();
for (int detail = maxDataLevel; detail >= 0; detail--) {
if (availableFiles[detail] == null) continue;
if (topValidDataLevel == Byte.MIN_VALUE) {
for (DataFile dataFile : availableFiles[detail]) {
if (dataFile.dataLevel > targetDataLevel) continue;
if (dataFile.dataType == ColumnDatatype.class || dataFile.dataType == FullDatatype.class) {
topValidDataLevel = LodUtil.max(topValidDataLevel, dataFile.dataLevel);
break;
}
}
}
if (topValidDataLevel == Byte.MIN_VALUE) continue;
DataFile singleCoveringColumnFile = null;
DataFile singleCoveringFullFile = null;
for (DataFile dataFile : availableFiles[detail]) {
if (dataFile.pos.getWidth().toBlock() == sectionPos.getWidth().toBlock()) {
if (dataFile.dataType == ColumnDatatype.class) {
singleCoveringColumnFile = dataFile;
break;
}
else if (dataFile.dataType == FullDatatype.class) {
singleCoveringFullFile = dataFile;
// Don't break as there may be a column file later.
}
} else if (dataFile.pos.getWidth().toBlock() > sectionPos.getWidth().toBlock()) {
if (dataFile.dataType == ColumnDatatype.class && singleCoveringColumnFile == null)
singleCoveringColumnFile = dataFile;
else if (dataFile.dataType == FullDatatype.class && singleCoveringFullFile == null)
singleCoveringFullFile = dataFile;
}
}
// First, try select single file that has enough width to cover the section
if (singleCoveringColumnFile != null) return Collections.singletonList(singleCoveringColumnFile);
if (singleCoveringFullFile != null) return Collections.singletonList(singleCoveringFullFile);
// If no single file covers the section, try to select all files without any duplicates
for (DataFile dataFile : availableFiles[detail]) {
boolean isDuplicate = false;
boolean isSet = false;
for (int i = 0; i < selectedFiles.size(); i++) {
DataFile selectedFile = selectedFiles.get(i);
if (selectedFile == null) continue;
if (selectedFile.pos.overlaps(dataFile.pos)) {
// Now, the already selected file muct have same or higher data level
// so, we just select the file with a position that covers the most area.
// Therefore, we choose the file with the higher section level.
if (selectedFile.pos.sectionDetail < dataFile.pos.sectionDetail) {
if (isSet) selectedFiles.set(i, null);
else selectedFiles.set(i, dataFile);
isSet = true;
} else {
LodUtil.assertTrue(!isSet); // We should not have encountered a smaller section level.
// This mean its completely covered by the selected file, so we can skip it.
isDuplicate = true;
break;
}
}
}
if (!isDuplicate && !isSet) selectedFiles.add(dataFile);
}
}
if (topValidDataLevel == Byte.MIN_VALUE) return Collections.emptyList();
selectedFiles.removeIf(Objects::isNull);
return selectedFiles;
}
};
public static final ColumnRenderLoader COLUMN_LAYER_LOADER = new ColumnRenderLoader();
public static final ColumnDataLoader COLUMN_DATA_LOADER = ColumnDataLoader.INSTANCE;
static {
LodQuadTree.registerLayerLoader(COLUMN_LAYER_LOADER, (byte) 7); // 7 or above
}
@Override
public DataSourceLoader getLatestLoader() {
return (DHLevel level, DhSectionPos sectionPos, InputStream data) -> loadFile(level, sectionPos, data, LATEST_VERSION);
}
@Override
public <T> T[] getData() {
return null;
return COLUMN_DATA_LOADER;
}
@Override
@@ -567,11 +398,12 @@ public class ColumnDatatype implements LodDataSource, RenderDataSource {
@Override
public byte getDetailOffset() {
return 0;
return SECTION_SIZE_OFFSET;
}
@Override
public boolean trySwapRenderBuffer(AtomicReference<RenderBuffer> referenceSlot) {
return false;
}
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/ColumnRenderLoader.java
+185
View File
@@ -0,0 +1,185 @@
package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFile;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.datatype.full.FullDatatype;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import com.seibel.lod.core.objects.a7.render.RenderDataSource;
import com.seibel.lod.core.objects.a7.render.RenderDataSourceLoader;
import com.seibel.lod.core.util.DetailDistanceUtil;
import com.seibel.lod.core.util.LodUtil;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
class ColumnRenderLoader extends RenderDataSourceLoader {
public ColumnRenderLoader() {
super(4);
}
@Override
public RenderDataSource construct(List<LodDataSource> dataSources, DhSectionPos sectionPos, DHLevel level) {
if (dataSources.size() == 0) return null;
// Check for direct casting
if (dataSources.size() == 1 && dataSources.get(0) instanceof ColumnDatatype
&& dataSources.get(0).getSectionPos().equals(sectionPos)
&& dataSources.get(0).getDataDetail() == sectionPos.sectionDetail - ColumnDatatype.SECTION_SIZE_OFFSET) {
// Directly using the data source as the render data source is possible.
return (ColumnDatatype) dataSources.get(0);
}
// Otherwise, we need to create a new render data source, and copy the data from the data sources.
ColumnDatatype renderDataSource = new ColumnDatatype(sectionPos,
DetailDistanceUtil.getMaxVerticalData(sectionPos.sectionDetail - ColumnDatatype.SECTION_SIZE_OFFSET),
level.getMinY());
boolean completeCopy = dataSources.get(0).getSectionPos().getWidth().toBlock() >= sectionPos.getWidth().toBlock();
if (completeCopy) {
// If there is only one data source, we need to insure on copy, we don't copy out of bounds as we
// may just need to copy partial section of the data source.
LodUtil.assertTrue(dataSources.size() == 1, "Expected only one data source for complete copy");
byte targetDataLevel = (byte) (sectionPos.sectionDetail - ColumnDatatype.SECTION_SIZE_OFFSET);
byte sourceDataLevel = dataSources.get(0).getDataDetail();
LodUtil.assertTrue(targetDataLevel >= sourceDataLevel);
if (dataSources.get(0) instanceof ColumnDatatype) {
ColumnDatatype dataSource = (ColumnDatatype) dataSources.get(0);
DhSectionPos srcPos = dataSource.getSectionPos();
// Note that in here, the source data level will be always < target section level
int trgX = sectionPos.getCorner().getX().toBlock();
int trgZ = sectionPos.getCorner().getZ().toBlock();
int trgMaxX = trgX + sectionPos.getWidth().toBlock() - 1;
int trgMaxZ = trgZ + sectionPos.getWidth().toBlock() - 1;
int trgXSizeInSrc = (trgX >> sourceDataLevel) - (trgMaxX >> sourceDataLevel) + 1;
int trgZSizeInSrc = (trgZ >> sourceDataLevel) - (trgMaxZ >> sourceDataLevel) + 1;
int trgXInSrc = (trgX >> sourceDataLevel) % srcPos.getWidth(sourceDataLevel).value;
int trgZInSrc = (trgZ >> sourceDataLevel) % srcPos.getWidth(sourceDataLevel).value;
ColumnQuadView srcView = dataSource.getDataInQuad(trgXInSrc, trgZInSrc, trgXSizeInSrc, trgZSizeInSrc);
ColumnQuadView trgView = renderDataSource.getFullQuad();
trgView.mergeMultiColumnFrom(srcView);
} else {
if (!(dataSources.get(0) instanceof FullDatatype))
throw new IllegalArgumentException("Unsupported data source type: " + dataSources.get(0).getClass().getName());
FullDatatype dataSource = (FullDatatype) dataSources.get(0);
DhSectionPos srcPos = dataSource.getSectionPos();
//TODO: Impl this
LodUtil.assertTrue(false, "Not implemented yet");
}
} else {
// If there are multiple data sources, we need to merge them into the target data source
for (LodDataSource dataSource : dataSources) {
byte targetDataLevel = (byte) (sectionPos.sectionDetail - ColumnDatatype.SECTION_SIZE_OFFSET);
byte sourceDataLevel = dataSource.getDataDetail();
DhSectionPos srcPos = dataSource.getSectionPos();
if (dataSource instanceof ColumnDatatype) {
ColumnDatatype clDataSource = (ColumnDatatype) dataSource;
// Note that targetDataLevel can be > source section level
int srcX = srcPos.getCorner().getX().toBlock();
int srcZ = srcPos.getCorner().getZ().toBlock();
int srcMaxX = srcX + srcPos.getWidth().toBlock() - 1;
int srcMaxZ = srcZ + srcPos.getWidth().toBlock() - 1;
int srcXSizeInTrg = (srcX >> targetDataLevel) - (srcMaxX >> targetDataLevel) + 1;
int srcZSizeInTrg = (srcZ >> targetDataLevel) - (srcMaxZ >> targetDataLevel) + 1;
int srcXInTrg = (srcX >> targetDataLevel) % ColumnDatatype.SECTION_SIZE;
int srcZInTrg = (srcZ >> targetDataLevel) % ColumnDatatype.SECTION_SIZE;
ColumnQuadView srcView = clDataSource.getFullQuad();
ColumnQuadView trgView = renderDataSource.getDataInQuad(srcXInTrg, srcZInTrg, srcXSizeInTrg, srcZSizeInTrg);
trgView.mergeMultiColumnFrom(srcView);
} else {
if (!(dataSource instanceof FullDatatype))
throw new IllegalArgumentException("Unsupported data source type: " + dataSource.getClass().getName());
FullDatatype flDataSource = (FullDatatype) dataSource;
//TODO: Impl this
LodUtil.assertTrue(false, "Not implemented yet");
}
}
}
return renderDataSource;
}
@Override
public List<DataFile> selectFiles(DhSectionPos sectionPos, DHLevel level, List<DataFile>[] availableFiles) {
byte targetDataLevel = (byte) (sectionPos.sectionDetail - ColumnDatatype.SECTION_SIZE_OFFSET);
//No support for loading higher than the target level yet.
byte maxDataLevel = LodUtil.min((byte) (availableFiles.length - 1), targetDataLevel);
byte topValidDataLevel = Byte.MIN_VALUE;
List<DataFile> selectedFiles = new LinkedList<>();
for (int detail = maxDataLevel; detail >= 0; detail--) {
if (availableFiles[detail] == null) continue;
if (topValidDataLevel == Byte.MIN_VALUE) {
for (DataFile dataFile : availableFiles[detail]) {
if (dataFile.dataLevel > targetDataLevel) continue;
if (dataFile.dataType == ColumnDatatype.class || dataFile.dataType == FullDatatype.class) {
topValidDataLevel = LodUtil.max(topValidDataLevel, dataFile.dataLevel);
break;
}
}
}
if (topValidDataLevel == Byte.MIN_VALUE) continue;
DataFile singleCoveringColumnFile = null;
DataFile singleCoveringFullFile = null;
for (DataFile dataFile : availableFiles[detail]) {
if (dataFile.pos.getWidth().toBlock() == sectionPos.getWidth().toBlock()) {
if (dataFile.dataType == ColumnDatatype.class) {
singleCoveringColumnFile = dataFile;
break;
} else if (dataFile.dataType == FullDatatype.class) {
singleCoveringFullFile = dataFile;
// Don't break as there may be a column file later.
}
} else if (dataFile.pos.getWidth().toBlock() > sectionPos.getWidth().toBlock()) {
if (dataFile.dataType == ColumnDatatype.class && singleCoveringColumnFile == null)
singleCoveringColumnFile = dataFile;
else if (dataFile.dataType == FullDatatype.class && singleCoveringFullFile == null)
singleCoveringFullFile = dataFile;
}
}
// First, try select single file that has enough width to cover the section
if (singleCoveringColumnFile != null) return Collections.singletonList(singleCoveringColumnFile);
if (singleCoveringFullFile != null) return Collections.singletonList(singleCoveringFullFile);
// If no single file covers the section, try to select all files without any duplicates
for (DataFile dataFile : availableFiles[detail]) {
boolean isDuplicate = false;
boolean isSet = false;
for (int i = 0; i < selectedFiles.size(); i++) {
DataFile selectedFile = selectedFiles.get(i);
if (selectedFile == null) continue;
if (selectedFile.pos.overlaps(dataFile.pos)) {
// Now, the already selected file muct have same or higher data level
// so, we just select the file with a position that covers the most area.
// Therefore, we choose the file with the higher section level.
if (selectedFile.pos.sectionDetail < dataFile.pos.sectionDetail) {
if (isSet) selectedFiles.set(i, null);
else selectedFiles.set(i, dataFile);
isSet = true;
} else {
LodUtil.assertTrue(!isSet); // We should not have encountered a smaller section level.
// This mean its completely covered by the selected file, so we can skip it.
isDuplicate = true;
break;
}
}
}
if (!isDuplicate && !isSet) selectedFiles.add(dataFile);
}
}
if (topValidDataLevel == Byte.MIN_VALUE) return Collections.emptyList();
selectedFiles.removeIf(Objects::isNull);
return selectedFiles;
}
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/DataSourceSaver.java
+24
View File
@@ -0,0 +1,24 @@
package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.objects.a7.DHLevel;
import com.seibel.lod.core.objects.a7.data.DataFileHandler;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
public abstract class DataSourceSaver extends DataSourceLoader {
public DataSourceSaver(Class<? extends LodDataSource> clazz, long datatypeId, byte loaderVersion) {
super(clazz, datatypeId, loaderVersion);
}
public abstract void saveData(DHLevel level, LodDataSource loadedData, DataOutputStream out) throws IOException;
// generate the default file path and file name based on various parameters.
// Ensure the file extension is '.lod'!
public File generateFilePathAndName(File levelFolderPath, DHLevel level, DhSectionPos sectionPos) {
return new File(levelFolderPath, String.format("%s_v%d-%s%s", clazz.getSimpleName(), loaderVersion,
sectionPos.serialize(), DataFileHandler.FILE_EXTENSION));
}
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/column/OldDataSourceLoader.java
+14
View File
@@ -0,0 +1,14 @@
package com.seibel.lod.core.objects.a7.datatype.column;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
public abstract class OldDataSourceLoader extends DataSourceLoader {
// Note: clazz can be null if the class no longer exists, as long as
// the datatypeId have not been changed or overwritten.
public OldDataSourceLoader(Class<? extends LodDataSource> clazz, long datatypeId, byte loaderVersion) {
super(clazz, datatypeId, loaderVersion);
}
abstract public DataSourceSaver getNewSaver();
}
src/main/java/com/seibel/lod/core/objects/a7/datatype/full/FullDatatype.java
+1
View File
@@ -1,5 +1,6 @@
package com.seibel.lod.core.objects.a7.datatype.full;
import com.seibel.lod.core.objects.a7.data.DataSourceLoader;
import com.seibel.lod.core.objects.a7.data.LodDataSource;
import com.seibel.lod.core.objects.a7.pos.DhSectionPos;
src/main/java/com/seibel/lod/core/objects/a7/pos/DhSectionPos.java
+24
View File
@@ -76,4 +76,28 @@ public class DhSectionPos {
public boolean overlaps(DhSectionPos other){
return getSectionBBoxPos().overlaps(other.getSectionBBoxPos());
}
@Override
public String toString() {
return "DhSectionPos{" +
"sectionDetail=" + sectionDetail +
", sectionX=" + sectionX +
", sectionZ=" + sectionZ +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DhSectionPos that = (DhSectionPos) o;
return sectionDetail == that.sectionDetail &&
sectionX == that.sectionX &&
sectionZ == that.sectionZ;
}
// Serialize() is different from toString() as this reqires it to NEVER be changed, and should be in a short format
public String serialize() {
return "[" + sectionDetail + ',' + sectionX + ',' + sectionZ + ']';
}
}