wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FINALLY fix the issue where some data isn't saved and etc

Browse Source
This commit is contained in:
TomTheFurry
2022-10-01 15:02:23 +08:00
parent b34778952a
commit 8d8de8692b
13 changed files with 459 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/lod/core/Initializer.java
+2 -1
View File
@@ -5,6 +5,7 @@ import com.seibel.lod.core.datatype.column.ColumnRenderLoader;
import com.seibel.lod.core.datatype.full.FullDataLoader;
import com.seibel.lod.core.datatype.full.SparseDataLoader;
import com.seibel.lod.api.DhApiMain;
import com.seibel.lod.core.datatype.full.SpottyDataLoader;
/**
* Handles first time Core setup.
@@ -19,7 +20,7 @@ public class Initializer
ColumnRenderLoader unused = new ColumnRenderLoader(); // Auto register into the loader system
FullDataLoader unused2 = new FullDataLoader(); // Auto register into the loader system
SparseDataLoader unused3 = new SparseDataLoader(); // Auto register
SpottyDataLoader unused4 = new SpottyDataLoader(); // Auto register
// link Core's config to the API
DhApiMain.configs = DhApiConfig.INSTANCE;
core/src/main/java/com/seibel/lod/core/datatype/IIncompleteDataSource.java
+11
View File
@@ -0,0 +1,11 @@
package com.seibel.lod.core.datatype;
import com.seibel.lod.core.datatype.full.accessor.SingleFullArrayView;
public interface IIncompleteDataSource extends ILodDataSource
{
void sampleFrom(ILodDataSource source);
ILodDataSource trySelfPromote();
// Return null if doesn't exist
SingleFullArrayView tryGet(int x, int z);
}
core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java
+3 -3
View File
@@ -1,8 +1,8 @@
package com.seibel.lod.core.datatype.column;
import com.seibel.lod.core.datatype.IIncompleteDataSource;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.full.FullDataSource;
import com.seibel.lod.core.datatype.full.SparseDataSource;
import com.seibel.lod.core.datatype.transform.FullToColumnTransformer;
import com.seibel.lod.core.level.IDhClientLevel;
import com.seibel.lod.core.datatype.ILodRenderSource;
@@ -31,8 +31,8 @@ public class ColumnRenderLoader extends AbstractRenderSourceLoader
public ILodRenderSource createRender(ILodDataSource dataSource, IDhClientLevel level) {
if (dataSource instanceof FullDataSource) {
return FullToColumnTransformer.transformFullDataToColumnData(level, (FullDataSource) dataSource);
} else if (dataSource instanceof SparseDataSource) {
return FullToColumnTransformer.transformSparseDataToColumnData(level, (SparseDataSource) dataSource);
} else if (dataSource instanceof IIncompleteDataSource) {
return FullToColumnTransformer.transformIncompleteDataToColumnData(level, (IIncompleteDataSource) dataSource);
}
LodUtil.assertNotReach();
return null;
core/src/main/java/com/seibel/lod/core/datatype/full/FullDataSource.java
+1 -1
View File
@@ -215,7 +215,7 @@ public class FullDataSource extends FullArrayView implements ILodDataSource
}
}
private FullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data)
FullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data)
{
super(mapping, data, SECTION_SIZE);
LodUtil.assertTrue(data.length == SECTION_SIZE * SECTION_SIZE);
core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java
+24 -26
View File
@@ -1,5 +1,6 @@
package com.seibel.lod.core.datatype.full;
import com.seibel.lod.core.datatype.IIncompleteDataSource;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.full.accessor.FullArrayView;
import com.seibel.lod.core.datatype.full.accessor.SingleFullArrayView;
@@ -14,7 +15,7 @@ import org.apache.logging.log4j.Logger;
import java.io.*;
import java.util.BitSet;
public class SparseDataSource implements ILodDataSource
public class SparseDataSource implements IIncompleteDataSource
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final byte SPARSE_UNIT_DETAIL = 4;
@@ -28,8 +29,8 @@ public class SparseDataSource implements ILodDataSource
protected final FullDataPointIdMap mapping;
private final DhSectionPos sectionPos;
private final FullArrayView[] sparseData;
private final int chunks;
private final int dataPerChunk;
final int chunks;
final int dataPerChunk;
private final DhLodPos chunkPos;
public boolean isEmpty = true;
@@ -56,6 +57,7 @@ public class SparseDataSource implements ILodDataSource
LodUtil.assertTrue(chunks*chunks == data.length);
sparseData = data;
chunkPos = sectionPos.getCorner(SPARSE_UNIT_DETAIL);
isEmpty = false;
this.mapping = mapping;
}
@@ -111,14 +113,26 @@ public class SparseDataSource implements ILodDataSource
return isEmpty;
}
public void sampleFrom(SparseDataSource sparseSource) {
DhSectionPos pos = sparseSource.sectionPos;
@Override
public void sampleFrom(ILodDataSource source) {
DhSectionPos pos = source.getSectionPos();
LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail);
LodUtil.assertTrue(pos.overlaps(sectionPos));
if (sparseSource.isEmpty) return;
if (source.isEmpty()) return;
if (source instanceof SparseDataSource) {
sampleFrom((SparseDataSource) source);
} else if (source instanceof FullDataSource) {
sampleFrom((FullDataSource) source);
} else {
LodUtil.assertNotReach();
}
}
private void sampleFrom(SparseDataSource sparseSource) {
DhSectionPos pos = sparseSource.getSectionPos();
isEmpty = false;
// Downsample needed
DhLodPos basePos = sectionPos.getCorner(SPARSE_UNIT_DETAIL);
DhLodPos dataPos = pos.getCorner(SPARSE_UNIT_DETAIL);
int offsetX = dataPos.x-basePos.x;
@@ -136,13 +150,10 @@ public class SparseDataSource implements ILodDataSource
}
}
}
public void sampleFrom(FullDataSource fullSource) {
private void sampleFrom(FullDataSource fullSource) {
DhSectionPos pos = fullSource.getSectionPos();
LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail);
LodUtil.assertTrue(pos.overlaps(sectionPos));
if (fullSource.isEmpty()) return;
isEmpty = false;
// Downsample needed
DhLodPos basePos = sectionPos.getCorner(SPARSE_UNIT_DETAIL);
DhLodPos dataPos = pos.getCorner(SPARSE_UNIT_DETAIL);
int coveredChunks = pos.getWidth(SPARSE_UNIT_DETAIL).value;
@@ -285,7 +296,7 @@ public class SparseDataSource implements ILodDataSource
}
}
public void applyToFullDataSource(FullDataSource dataSource) {
private void applyToFullDataSource(FullDataSource dataSource) {
LodUtil.assertTrue(dataSource.getSectionPos().equals(sectionPos));
LodUtil.assertTrue(dataSource.getDataDetail() == getDataDetail());
for (int x = 0; x<chunks; x++) {
@@ -300,19 +311,6 @@ public class SparseDataSource implements ILodDataSource
}
}
public ILodDataSource promote(ILodDataSource generatedData) {
if (!(generatedData instanceof FullDataSource) && !(generatedData instanceof SparseDataSource))
throw new UnsupportedOperationException("Requires FullDataSource for the promotion!");
if (generatedData instanceof FullDataSource) {
applyToFullDataSource((FullDataSource) generatedData);
return generatedData;
} else {
LodUtil.assertToDo(); //TODO
return null;
}
}
public ILodDataSource trySelfPromote() {
if (isEmpty) return this;
for (FullArrayView array : sparseData) {
core/src/main/java/com/seibel/lod/core/datatype/full/SpottyDataLoader.java
+21
View File
@@ -0,0 +1,21 @@
package com.seibel.lod.core.datatype.full;
import com.seibel.lod.core.datatype.AbstractDataSourceLoader;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.file.datafile.DataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import java.io.IOException;
import java.io.InputStream;
public class SpottyDataLoader extends AbstractDataSourceLoader
{
public SpottyDataLoader() {
super(SpottyDataSource.class, SpottyDataSource.TYPE_ID, new byte[]{SpottyDataSource.LATEST_VERSION});
}
@Override
public ILodDataSource loadData(DataMetaFile dataFile, InputStream data, IDhLevel level) throws IOException {
return SpottyDataSource.loadData(dataFile, data, level);
}
}
core/src/main/java/com/seibel/lod/core/datatype/full/SpottyDataSource.java
+287
View File
@@ -0,0 +1,287 @@
package com.seibel.lod.core.datatype.full;
import com.seibel.lod.core.datatype.IIncompleteDataSource;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.full.accessor.FullArrayView;
import com.seibel.lod.core.datatype.full.accessor.SingleFullArrayView;
import com.seibel.lod.core.file.datafile.DataMetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.pos.DhLodPos;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.util.LodUtil;
import com.seibel.lod.core.util.objects.UnclosableInputStream;
import org.apache.logging.log4j.Logger;
import java.io.*;
import java.util.BitSet;
public class SpottyDataSource extends FullArrayView implements IIncompleteDataSource { // 1 chunk
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final byte SECTION_SIZE_OFFSET = 6;
public static final int SECTION_SIZE = 1 << SECTION_SIZE_OFFSET;
public static final byte LATEST_VERSION = 0;
public static final long TYPE_ID = "SpottyDataSource".hashCode();
private final DhSectionPos sectionPos;
private boolean isEmpty = true;
private final BitSet isColumnNotEmpty;
protected SpottyDataSource(DhSectionPos sectionPos) {
super(new FullDataPointIdMap(), new long[SECTION_SIZE*SECTION_SIZE][0], SECTION_SIZE);
LodUtil.assertTrue(sectionPos.sectionDetail > SparseDataSource.MAX_SECTION_DETAIL);
this.sectionPos = sectionPos;
isColumnNotEmpty = new BitSet(SECTION_SIZE*SECTION_SIZE);
}
@Override
public DhSectionPos getSectionPos() {
return sectionPos;
}
@Override
public byte getDataDetail() {
return (byte) (sectionPos.sectionDetail-SECTION_SIZE_OFFSET);
}
@Override
public byte getDataVersion() {
return LATEST_VERSION;
}
@Override
public void update(ChunkSizedData data) {
LodUtil.assertTrue(sectionPos.getSectionBBoxPos().overlaps(data.getBBoxLodPos()));
if (data.dataDetail == 0 && getDataDetail() >= 4) {
//FIXME: TEMPORARY
int chunkPerFull = 1 << (getDataDetail() - 4);
if (data.x % chunkPerFull != 0 || data.z % chunkPerFull != 0) return;
DhLodPos baseOffset = sectionPos.getCorner(getDataDetail());
DhLodPos dataOffset = data.getBBoxLodPos().convertUpwardsTo(getDataDetail());
int offsetX = dataOffset.x - baseOffset.x;
int offsetZ = dataOffset.z - baseOffset.z;
LodUtil.assertTrue(offsetX >= 0 && offsetX < SECTION_SIZE && offsetZ >= 0 && offsetZ < SECTION_SIZE);
isEmpty = false;
data.get(0,0).deepCopyTo(get(offsetX, offsetZ));
} else {
LodUtil.assertNotReach();
//TODO;
}
}
@Override
public boolean isEmpty() {
return isEmpty;
}
public void markNotEmpty() {
isEmpty = false;
}
@Override
public void saveData(IDhLevel level, DataMetaFile file, OutputStream dataStream) throws IOException {
DataOutputStream dos = new DataOutputStream(dataStream); // DO NOT CLOSE
{
dos.writeInt(getDataDetail());
dos.writeInt(size);
dos.writeInt(level.getMinY());
if (isEmpty) {
dos.writeInt(0x00000001);
return;
}
// Is column not empty
dos.writeInt(0xFFFFFFFF);
byte[] bytes = isColumnNotEmpty.toByteArray();
dos.writeInt(bytes.length);
dos.write(bytes);
// Data array content
dos.writeInt(0xFFFFFFFF);
for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1))
{
dos.writeByte(dataArrays[i].length);
if (dataArrays[i].length == 0) continue;
for (long l : dataArrays[i]) {
dos.writeLong(l);
}
}
// Id mapping
dos.writeInt(0xFFFFFFFF);
mapping.serialize(dos);
dos.writeInt(0xFFFFFFFF);
}
}
public static SpottyDataSource loadData(DataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException {
DataInputStream dos = new DataInputStream(dataStream); // DO NOT CLOSE
{
int dataDetail = dos.readInt();
if(dataDetail != dataFile.metaData.dataLevel)
throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
int size = dos.readInt();
if (size != SECTION_SIZE)
throw new IOException(LodUtil.formatLog(
"Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE));
int minY = dos.readInt();
if (minY != level.getMinY())
LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY());
int end = dos.readInt();
// Data array length
if (end == 0x00000001) {
// Section is empty
return new SpottyDataSource(dataFile.pos);
}
// Is column not empty
if (end != 0xFFFFFFFF) throw new IOException("invalid header end guard");
int length = dos.readInt();
if (length < 0 || length > (SECTION_SIZE*SECTION_SIZE/8+64)*2)
throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})",
length, 1, SECTION_SIZE*SECTION_SIZE/8+63));
byte[] bytes = dos.readNBytes(length);
BitSet isColumnNotEmpty = BitSet.valueOf(bytes);
// Data array content
long[][] data = new long[SECTION_SIZE*SECTION_SIZE][];
end = dos.readInt();
if (end != 0xFFFFFFFF) throw new IOException("invalid spotty flag end guard");
for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1))
{
long[] array = new long[dos.readByte()];
for (int j = 0; j < array.length; j++) {
array[j] = dos.readLong();
}
data[i] = array;
}
// Id mapping
end = dos.readInt();
if (end != 0xFFFFFFFF) throw new IOException("invalid data content end guard");
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(new UnclosableInputStream(dos));
end = dos.readInt();
if (end != 0xFFFFFFFF) throw new IOException("invalid id mapping end guard");
return new SpottyDataSource(dataFile.pos, mapping, isColumnNotEmpty, data);
}
}
private SpottyDataSource(DhSectionPos pos, FullDataPointIdMap mapping, BitSet isColumnNotEmpty, long[][] data) {
super(mapping, data, SECTION_SIZE);
LodUtil.assertTrue(data.length == SECTION_SIZE*SECTION_SIZE);
this.sectionPos = pos;
this.isColumnNotEmpty = isColumnNotEmpty;
isEmpty = false;
}
public static SpottyDataSource createEmpty(DhSectionPos pos) {
return new SpottyDataSource(pos);
}
public static boolean neededForPosition(DhSectionPos posToWrite, DhSectionPos posToTest) {
if (!posToWrite.overlaps(posToTest)) return false;
if (posToTest.sectionDetail > posToWrite.sectionDetail) return false;
if (posToWrite.sectionDetail - posToTest.sectionDetail <= SECTION_SIZE_OFFSET) return true;
byte sectPerData = (byte) (1 << (posToWrite.sectionDetail - posToTest.sectionDetail - SECTION_SIZE_OFFSET));
return posToTest.sectionX % sectPerData == 0 && posToTest.sectionZ % sectPerData == 0;
}
@Override
public void sampleFrom(ILodDataSource source) {
DhSectionPos pos = source.getSectionPos();
LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail);
LodUtil.assertTrue(pos.overlaps(sectionPos));
if (source.isEmpty()) return;
if (source instanceof SparseDataSource) {
sampleFrom((SparseDataSource) source);
} else if (source instanceof FullDataSource) {
sampleFrom((FullDataSource) source);
} else {
LodUtil.assertNotReach();
}
}
private void sampleFrom(SparseDataSource sparseSource) {
DhSectionPos pos = sparseSource.getSectionPos();
isEmpty = false;
if (getDataDetail() > sectionPos.sectionDetail) {
DhLodPos basePos = sectionPos.getCorner(getDataDetail());
DhLodPos dataPos = pos.getCorner(getDataDetail());
int offsetX = dataPos.x - basePos.x;
int offsetZ = dataPos.z - basePos.z;
LodUtil.assertTrue(offsetX >= 0 && offsetX < SECTION_SIZE && offsetZ >= 0 && offsetZ < SECTION_SIZE);
int chunksPerData = 1 << (getDataDetail() - SparseDataSource.SPARSE_UNIT_DETAIL);
int dataSpan = sectionPos.getWidth(getDataDetail()).value;
for (int ox = 0; ox < dataSpan; ox++) {
for (int oz = 0; oz < dataSpan; oz++) {
SingleFullArrayView column = sparseSource.tryGet(
ox * chunksPerData * sparseSource.dataPerChunk,
oz * chunksPerData * sparseSource.dataPerChunk);
if (column != null) {
column.deepCopyTo(get(offsetX + ox, offsetZ + oz));
isColumnNotEmpty.set((offsetX + ox) * SECTION_SIZE + offsetZ + oz, true);
}
}
}
} else {
DhLodPos dataPos = pos.getSectionBBoxPos();
int lowerSectionsPerData = sectionPos.getWidth(dataPos.detail).value;
if (dataPos.x % lowerSectionsPerData != 0 || dataPos.z % lowerSectionsPerData != 0) return;
DhLodPos basePos = sectionPos.getCorner(getDataDetail());
dataPos = dataPos.convertUpwardsTo(getDataDetail());
int offsetX = dataPos.x - basePos.x;
int offsetZ = dataPos.z - basePos.z;
SingleFullArrayView column = sparseSource.tryGet(0, 0);
if (column != null) {
column.deepCopyTo(get(offsetX, offsetZ));
isColumnNotEmpty.set(offsetX * SECTION_SIZE + offsetZ, true);
}
}
}
private void sampleFrom(FullDataSource fullSource) {
DhSectionPos pos = fullSource.getSectionPos();
isEmpty = false;
downsampleFrom(fullSource);
if (getDataDetail() > sectionPos.sectionDetail) {
DhLodPos basePos = sectionPos.getCorner(getDataDetail());
DhLodPos dataPos = pos.getCorner(getDataDetail());
int offsetX = dataPos.x - basePos.x;
int offsetZ = dataPos.z - basePos.z;
int dataSpan = sectionPos.getWidth(getDataDetail()).value;
for (int ox = 0; ox < dataSpan; ox++) {
for (int oz = 0; oz < dataSpan; oz++) {
isColumnNotEmpty.set((offsetX + ox) * SECTION_SIZE + offsetZ + oz, true);
}
}
} else {
DhLodPos dataPos = pos.getSectionBBoxPos();
int lowerSectionsPerData = sectionPos.getWidth(dataPos.detail).value;
if (dataPos.x % lowerSectionsPerData != 0 || dataPos.z % lowerSectionsPerData != 0) return;
DhLodPos basePos = sectionPos.getCorner(getDataDetail());
dataPos = dataPos.convertUpwardsTo(getDataDetail());
int offsetX = dataPos.x - basePos.x;
int offsetZ = dataPos.z - basePos.z;
isColumnNotEmpty.set(offsetX * SECTION_SIZE + offsetZ, true);
}
}
@Override
public ILodDataSource trySelfPromote() {
if (isEmpty) return this;
if (isColumnNotEmpty.cardinality() != SECTION_SIZE * SECTION_SIZE) return this;
return new FullDataSource(sectionPos, mapping, dataArrays);
}
@Override
public SingleFullArrayView tryGet(int x, int z) {
return isColumnNotEmpty.get(x * SECTION_SIZE + z) ? get(x, z) : null;
}
}
core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java
+3 -2
View File
@@ -1,5 +1,6 @@
package com.seibel.lod.core.datatype.transform;
import com.seibel.lod.core.datatype.IIncompleteDataSource;
import com.seibel.lod.core.datatype.ILodRenderSource;
import com.seibel.lod.core.datatype.column.accessor.ColumnFormat;
import com.seibel.lod.core.datatype.column.ColumnRenderSource;
@@ -66,12 +67,12 @@ public class FullToColumnTransformer {
return columnSource;
}
public static ILodRenderSource transformSparseDataToColumnData(IDhClientLevel level, SparseDataSource data) {
public static ILodRenderSource transformIncompleteDataToColumnData(IDhClientLevel level, IIncompleteDataSource data) {
final DhSectionPos pos = data.getSectionPos();
final byte dataDetail = data.getDataDetail();
final int vertSize = Config.Client.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(data.getDataDetail());
final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY());
if (data.isEmpty) return columnSource;
if (data.isEmpty()) return columnSource;
columnSource.markNotEmpty();
if (dataDetail == columnSource.getDataDetail()) {
core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java
+24 -16
View File
@@ -1,10 +1,12 @@
package com.seibel.lod.core.file.datafile;
import com.google.common.collect.HashMultimap;
import com.seibel.lod.core.datatype.IIncompleteDataSource;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.full.ChunkSizedData;
import com.seibel.lod.core.datatype.full.FullDataSource;
import com.seibel.lod.core.datatype.full.SparseDataSource;
import com.seibel.lod.core.datatype.full.SpottyDataSource;
import com.seibel.lod.core.file.MetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhLodPos;
@@ -255,10 +257,17 @@ public class DataFileHandler implements IDataSourceProvider {
}
@Override
public long getLatestCacheVersion(DhSectionPos sectionPos) {
public long getCacheVersion(DhSectionPos sectionPos) {
DataMetaFile file = files.get(sectionPos);
if (file == null) return 0;
return file.getDataVersion();
return file.getCacheVersion();
}
@Override
public boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion) {
DataMetaFile file = files.get(sectionPos);
if (file == null) return cacheVersion >= 0;
return file.isCacheVersionValid(cacheVersion);
}
@Override
@@ -270,27 +279,25 @@ public class DataFileHandler implements IDataSourceProvider {
LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty());
if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) {
// None exist.
SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
return CompletableFuture.completedFuture(dataSource);
IIncompleteDataSource incompleteDataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ?
SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos);
return CompletableFuture.completedFuture(incompleteDataSource);
} else {
for (DhSectionPos missingPos : missing) {
DataMetaFile newfile = atomicGetOrMakeFile(missingPos);
if (newfile != null) existFiles.add(newfile);
}
final ArrayList<CompletableFuture<Void>> futures = new ArrayList<>(existFiles.size());
final SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
final IIncompleteDataSource dataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ?
SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos);
for (DataMetaFile f : existFiles) {
futures.add(f.loadOrGetCached()
.exceptionally((ex) -> null)
.thenAccept((data) -> {
if (data != null) {
if (data instanceof SparseDataSource)
dataSource.sampleFrom((SparseDataSource) data);
else if (data instanceof FullDataSource)
dataSource.sampleFrom((FullDataSource) data);
else LodUtil.assertNotReach();
LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos);
dataSource.sampleFrom(data);
}
})
);
@@ -305,8 +312,8 @@ public class DataFileHandler implements IDataSourceProvider {
Consumer<ILodDataSource> onUpdated, Function<ILodDataSource, Boolean> updater) {
boolean changed = updater.apply(source);
if (changed) metaData.dataVersion.incrementAndGet();
if (source instanceof SparseDataSource) {
ILodDataSource newSource = ((SparseDataSource) source).trySelfPromote();
if (source instanceof IIncompleteDataSource) {
ILodDataSource newSource = ((IIncompleteDataSource) source).trySelfPromote();
changed |= newSource != source;
source = newSource;
}
@@ -314,12 +321,13 @@ public class DataFileHandler implements IDataSourceProvider {
return source;
}
@Override
public CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated) {
public CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated) {
return CompletableFuture.supplyAsync(() -> {
ILodDataSource sourceLocal = source;
boolean changed = updater.apply(sourceLocal);
if (sourceLocal instanceof SparseDataSource) {
ILodDataSource newSource = ((SparseDataSource) sourceLocal).trySelfPromote();
if (changed) metaData.dataVersion.incrementAndGet();
if (sourceLocal instanceof IIncompleteDataSource) {
ILodDataSource newSource = ((IIncompleteDataSource) sourceLocal).trySelfPromote();
changed |= newSource != sourceLocal;
sourceLocal = newSource;
}
core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java
+42 -17
View File
@@ -17,6 +17,7 @@ import com.seibel.lod.core.file.MetaFile;
import com.seibel.lod.core.level.IDhLevel;
import com.seibel.lod.core.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.AtomicsUtil;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
@@ -48,7 +49,7 @@ public class DataMetaFile extends MetaFile
GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue();
// ===========================
private final AtomicBoolean inCacheWriteAccessAsserter = new AtomicBoolean(false);
private AtomicReference<CompletableFuture<ILodDataSource>> inCacheWriteAccessFuture = new AtomicReference<>(null);
// ===Object lifetime stuff===
private static final ReferenceQueue<ILodDataSource> lifeCycleDebugQueue = new ReferenceQueue<>();
@@ -105,16 +106,29 @@ public class DataMetaFile extends MetaFile
}
}
public long getDataVersion() {
public long getCacheVersion() {
debugCheck();
MetaData getData = metaData;
return getData == null ? 0 : metaData.dataVersion.get();
}
public boolean isCacheVersionValid(long cacheVersion) {
debugCheck();
boolean noWrite = writeQueue.get().queue.isEmpty();
if (!noWrite) {
return false;
} else {
MetaData getData = metaData;
//NOTE: Do this instead of direct compare so values that wrapped around still works correctly.
return (getData == null ? 0 : metaData.dataVersion.get()) - cacheVersion <= 0;
}
}
public void addToWriteQueue(ChunkSizedData datatype) {
debugCheck();
DhLodPos chunkPos = new DhLodPos((byte) (datatype.dataDetail + 4), datatype.x, datatype.z);
LodUtil.assertTrue(pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos {} doesn't overlap with section {}", chunkPos, pos);
//LOGGER.info("Write Chunk {} to file {}", chunkPos, pos);
GuardedMultiAppendQueue queue = writeQueue.get();
// Using read lock is OK, because the queue's underlying data structure is thread-safe.
@@ -175,7 +189,7 @@ public class DataMetaFile extends MetaFile
throw new CompletionException(e);
}
// Apply the write queue
LodUtil.assertTrue(!inCacheWriteAccessAsserter.get(),"No one should be writing to the cache while we are in the process of " +
LodUtil.assertTrue(inCacheWriteAccessFuture.get() == null,"No one should be writing to the cache while we are in the process of " +
"loading one into the cache! Is this a deadlock?");
data = handler.onDataFileLoaded(data, metaData, this::saveChanges, this::applyWriteQueue);
// Finally, return the data.
@@ -207,7 +221,7 @@ public class DataMetaFile extends MetaFile
// "unchecked": Suppress casting of CompletableFuture<?> to CompletableFuture<LodDataSource>
// "PointlessBooleanExpression": Suppress explicit (boolean == false) check for more understandable CAS operation code.
@SuppressWarnings({"unchecked", "PointlessBooleanExpression"})
@SuppressWarnings({"unchecked"})
private CompletableFuture<ILodDataSource> _readCached(Object obj) {
// Has file cached in RAM and not freed yet.
if ((obj instanceof SoftReference<?>)) {
@@ -219,23 +233,33 @@ public class DataMetaFile extends MetaFile
// that will be applying the changes to the cache.
if (!isEmpty) {
// Do a CAS on inCacheWriteLock to ensure that we are the only thread that is writing to the cache,
// or if we fail, then that means someone else is already doing it, and we can just continue.
// FIXME: Should we return a future that waits for the write to be done for CAS fail? Or should we just return the
// cached data that doesn't have all writes done immediately?
// The latter give us immediate access to the data, but we need to ensure concurrent reads and
// writes doesn't cause unexpected behavior down the line.
// For now, I'll go for the latter option and just hope nothing goes wrong...
if (inCacheWriteAccessAsserter.getAndSet(true) == false) {
// or if we fail, then that means someone else is already doing it, and we can just return the future
CompletableFuture<ILodDataSource> future = new CompletableFuture<>();
CompletableFuture<ILodDataSource> cas = AtomicsUtil.compareAndExchange(inCacheWriteAccessFuture, null, future);
if (cas == null) {
try {
return handler.onDataFileRefresh((ILodDataSource) inner, this::applyWriteQueue, this::saveChanges);
data.set(future);
handler.onDataFileRefresh((ILodDataSource) inner, metaData, this::applyWriteQueue, this::saveChanges).handle((v, e) -> {
if (e != null) {
LOGGER.error("Error refreshing data {}: ", pos, e);
future.complete(null);
data.set(null);
} else {
future.complete(v);
new DataObjTracker(v);
data.set(new SoftReference<>(v));
}
inCacheWriteAccessFuture.set(null);
return v;
});
return future;
} catch (Exception e) {
LOGGER.error("Error while applying changes to LodDataSource at {}: ", pos, e);
} finally {
inCacheWriteAccessAsserter.set(false);
LOGGER.error("Error while doing refreshes to LodDataSource at {}: ", pos, e);
return CompletableFuture.completedFuture((ILodDataSource) inner);
}
} else {
// or, return the cached data. FIXME: See above.
return CompletableFuture.completedFuture((ILodDataSource) inner);
// or, return the future that will be completed when the write is done.
return cas;
}
} else {
// or, return the cached data.
@@ -270,6 +294,7 @@ public class DataMetaFile extends MetaFile
if (path.exists()) if (!path.delete()) LOGGER.warn("Failed to delete data file at {}", path);
doesFileExist = false;
} else {
LOGGER.info("Saving data file of {}", data.getSectionPos());
try {
// Write/Update data
LodUtil.assertTrue(metaData != null);
core/src/main/java/com/seibel/lod/core/file/datafile/GeneratedDataFileHandler.java
+9 -8
View File
@@ -1,9 +1,10 @@
package com.seibel.lod.core.file.datafile;
import com.seibel.lod.core.datatype.IIncompleteDataSource;
import com.seibel.lod.core.datatype.ILodDataSource;
import com.seibel.lod.core.datatype.full.ChunkSizedData;
import com.seibel.lod.core.datatype.full.FullDataSource;
import com.seibel.lod.core.datatype.full.SparseDataSource;
import com.seibel.lod.core.datatype.full.SpottyDataSource;
import com.seibel.lod.core.generation.GenerationQueue;
import com.seibel.lod.core.level.IDhServerLevel;
import com.seibel.lod.core.pos.DhSectionPos;
@@ -99,7 +100,8 @@ public class GeneratedDataFileHandler extends DataFileHandler {
LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty());
if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) {
// None exist.
SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
IIncompleteDataSource dataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ?
SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos);
GenerationQueue getQueue = queue.get();
GenTask task = new GenTask(pos, new WeakReference<>(dataSource));
genQueue.put(dataSource, task);
@@ -126,18 +128,17 @@ public class GeneratedDataFileHandler extends DataFileHandler {
if (newfile != null) existFiles.add(newfile);
}
final ArrayList<CompletableFuture<Void>> futures = new ArrayList<>(existFiles.size());
final SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
final IIncompleteDataSource dataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ?
SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos);
LOGGER.debug("Creating {} from sampling {} files: {}", pos, existFiles.size(), existFiles);
for (DataMetaFile f : existFiles) {
futures.add(f.loadOrGetCached()
.exceptionally((ex) -> null)
.thenAccept((data) -> {
if (data != null) {
if (data instanceof SparseDataSource)
dataSource.sampleFrom((SparseDataSource) data);
else if (data instanceof FullDataSource)
dataSource.sampleFrom((FullDataSource) data);
else LodUtil.assertNotReach();
LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos);
dataSource.sampleFrom(data);
}
})
);
core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java
+3 -2
View File
@@ -19,11 +19,12 @@ public interface IDataSourceProvider extends AutoCloseable {
void write(DhSectionPos sectionPos, ChunkSizedData chunkData);
CompletableFuture<Void> flushAndSave();
long getLatestCacheVersion(DhSectionPos sectionPos);
long getCacheVersion(DhSectionPos sectionPos);
boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion);
CompletableFuture<ILodDataSource> onCreateDataFile(DataMetaFile file);
ILodDataSource onDataFileLoaded(ILodDataSource source, MetaFile.MetaData metaData, Consumer<ILodDataSource> onUpdated, Function<ILodDataSource, Boolean> updater);
CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated);
CompletableFuture<ILodDataSource> onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function<ILodDataSource, Boolean> updater, Consumer<ILodDataSource> onUpdated);
File computeDataFilePath(DhSectionPos pos);
Executor getIOExecutor();
core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java
+29 -32
View File
@@ -17,6 +17,7 @@ import com.seibel.lod.core.config.Config;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import org.lwjgl.system.CallbackI;
import java.io.File;
import java.io.IOException;
@@ -204,33 +205,31 @@ public class RenderFileHandler implements IRenderSourceProvider {
if (cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null) return;
final WeakReference<ILodRenderSource> dataRef = new WeakReference<>(data);
CompletableFuture<ILodDataSource> dataFuture = dataSourceProvider.read(data.getSectionPos());
final long version = dataSourceProvider.getLatestCacheVersion(data.getSectionPos());
DataRenderTransformer.asyncTransformDataSource(
dataFuture.thenApply((d) -> {
if (dataRef.get() == null) throw new UncheckedInterruptedException();
LodUtil.assertTrue(d != null);
return d;
}).exceptionally((ex) -> {
if (ex != null)
LOGGER.error("Uncaught exception when getting data for updateCache()", ex);
return null;
})
, level)
.thenAccept((newData) -> write(dataRef.get(), file, newData, version))
.exceptionally((ex) -> {
if (!UncheckedInterruptedException.isThrowableInterruption(ex))
LOGGER.error("Exception when updating render file using data source: ", ex);
return null;
}).thenRun(() -> cacheRecreationGuards.remove(file.pos));
dataFuture = dataFuture.thenApply((d) -> {
if (dataRef.get() == null) throw new UncheckedInterruptedException();
LodUtil.assertTrue(d != null);
return d;
}).exceptionally((ex) -> {
if (ex != null)
LOGGER.error("Uncaught exception when getting data for updateCache()", ex);
return null;
});
LOGGER.info("Recreating cache for {}", data.getSectionPos());
DataRenderTransformer.asyncTransformDataSource(dataFuture , level)
.thenAccept((newData) -> write(dataRef.get(), file, newData, dataSourceProvider.getCacheVersion(data.getSectionPos())))
.exceptionally((ex) -> {
if (!UncheckedInterruptedException.isThrowableInterruption(ex))
LOGGER.error("Exception when updating render file using data source: ", ex);
return null;
}).thenRun(() -> cacheRecreationGuards.remove(file.pos));
}
public ILodRenderSource onRenderFileLoaded(ILodRenderSource data, RenderMetaFile file) {
long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos);
//NOTE: Do this instead of direct compare so values that wrapped around still works correctly.
if (newCacheVersion - file.metaData.dataVersion.get() <= 0)
return data;
updateCache(data, file);
if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) {
updateCache(data, file);
}
return data;
}
@@ -253,10 +252,9 @@ public class RenderFileHandler implements IRenderSourceProvider {
}
public void onReadRenderSourceFromCache(RenderMetaFile file, ILodRenderSource data) {
long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos);
//NOTE: Do this instead of direct compare so values that wrapped around still works correctly.
if (newCacheVersion - file.metaData.dataVersion.get() > 0)
if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) {
updateCache(data, file);
}
}
public boolean refreshRenderSource(ILodRenderSource source) {
@@ -268,12 +266,11 @@ public class RenderFileHandler implements IRenderSourceProvider {
}
LodUtil.assertTrue(file != null);
LodUtil.assertTrue(file.metaData != null);
long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos);
//NOTE: Do this instead of direct compare so values that wrapped around still works correctly.
if (newCacheVersion - file.metaData.dataVersion.get() <= 0)
return false;
updateCache(source, file);
return true;
if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) {
updateCache(source, file);
return true;
}
return false;
}
}