diff --git a/core/src/main/java/com/seibel/lod/core/Initializer.java b/core/src/main/java/com/seibel/lod/core/Initializer.java index 453c020b3..ff038d29e 100644 --- a/core/src/main/java/com/seibel/lod/core/Initializer.java +++ b/core/src/main/java/com/seibel/lod/core/Initializer.java @@ -5,6 +5,7 @@ import com.seibel.lod.core.datatype.column.ColumnRenderLoader; import com.seibel.lod.core.datatype.full.FullDataLoader; import com.seibel.lod.core.datatype.full.SparseDataLoader; import com.seibel.lod.api.DhApiMain; +import com.seibel.lod.core.datatype.full.SpottyDataLoader; /** * Handles first time Core setup. @@ -19,7 +20,7 @@ public class Initializer ColumnRenderLoader unused = new ColumnRenderLoader(); // Auto register into the loader system FullDataLoader unused2 = new FullDataLoader(); // Auto register into the loader system SparseDataLoader unused3 = new SparseDataLoader(); // Auto register - + SpottyDataLoader unused4 = new SpottyDataLoader(); // Auto register // link Core's config to the API DhApiMain.configs = DhApiConfig.INSTANCE; diff --git a/core/src/main/java/com/seibel/lod/core/datatype/IIncompleteDataSource.java b/core/src/main/java/com/seibel/lod/core/datatype/IIncompleteDataSource.java new file mode 100644 index 000000000..9e4e9158a --- /dev/null +++ b/core/src/main/java/com/seibel/lod/core/datatype/IIncompleteDataSource.java @@ -0,0 +1,11 @@ +package com.seibel.lod.core.datatype; + +import com.seibel.lod.core.datatype.full.accessor.SingleFullArrayView; + +public interface IIncompleteDataSource extends ILodDataSource +{ + void sampleFrom(ILodDataSource source); + ILodDataSource trySelfPromote(); + // Return null if doesn't exist + SingleFullArrayView tryGet(int x, int z); +} diff --git a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java index cffad130d..9d6b13a31 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderLoader.java @@ -1,8 +1,8 @@ package com.seibel.lod.core.datatype.column; +import com.seibel.lod.core.datatype.IIncompleteDataSource; import com.seibel.lod.core.datatype.ILodDataSource; import com.seibel.lod.core.datatype.full.FullDataSource; -import com.seibel.lod.core.datatype.full.SparseDataSource; import com.seibel.lod.core.datatype.transform.FullToColumnTransformer; import com.seibel.lod.core.level.IDhClientLevel; import com.seibel.lod.core.datatype.ILodRenderSource; @@ -31,8 +31,8 @@ public class ColumnRenderLoader extends AbstractRenderSourceLoader public ILodRenderSource createRender(ILodDataSource dataSource, IDhClientLevel level) { if (dataSource instanceof FullDataSource) { return FullToColumnTransformer.transformFullDataToColumnData(level, (FullDataSource) dataSource); - } else if (dataSource instanceof SparseDataSource) { - return FullToColumnTransformer.transformSparseDataToColumnData(level, (SparseDataSource) dataSource); + } else if (dataSource instanceof IIncompleteDataSource) { + return FullToColumnTransformer.transformIncompleteDataToColumnData(level, (IIncompleteDataSource) dataSource); } LodUtil.assertNotReach(); return null; diff --git a/core/src/main/java/com/seibel/lod/core/datatype/full/FullDataSource.java b/core/src/main/java/com/seibel/lod/core/datatype/full/FullDataSource.java index cd66e8c52..3c4824a0b 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/full/FullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/full/FullDataSource.java @@ -215,7 +215,7 @@ public class FullDataSource extends FullArrayView implements ILodDataSource } } - private FullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data) + FullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data) { super(mapping, data, SECTION_SIZE); LodUtil.assertTrue(data.length == SECTION_SIZE * SECTION_SIZE); diff --git a/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java b/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java index 07761e309..6f6328399 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java @@ -1,5 +1,6 @@ package com.seibel.lod.core.datatype.full; +import com.seibel.lod.core.datatype.IIncompleteDataSource; import com.seibel.lod.core.datatype.ILodDataSource; import com.seibel.lod.core.datatype.full.accessor.FullArrayView; import com.seibel.lod.core.datatype.full.accessor.SingleFullArrayView; @@ -14,7 +15,7 @@ import org.apache.logging.log4j.Logger; import java.io.*; import java.util.BitSet; -public class SparseDataSource implements ILodDataSource +public class SparseDataSource implements IIncompleteDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); public static final byte SPARSE_UNIT_DETAIL = 4; @@ -28,8 +29,8 @@ public class SparseDataSource implements ILodDataSource protected final FullDataPointIdMap mapping; private final DhSectionPos sectionPos; private final FullArrayView[] sparseData; - private final int chunks; - private final int dataPerChunk; + final int chunks; + final int dataPerChunk; private final DhLodPos chunkPos; public boolean isEmpty = true; @@ -56,6 +57,7 @@ public class SparseDataSource implements ILodDataSource LodUtil.assertTrue(chunks*chunks == data.length); sparseData = data; chunkPos = sectionPos.getCorner(SPARSE_UNIT_DETAIL); + isEmpty = false; this.mapping = mapping; } @@ -111,14 +113,26 @@ public class SparseDataSource implements ILodDataSource return isEmpty; } - public void sampleFrom(SparseDataSource sparseSource) { - DhSectionPos pos = sparseSource.sectionPos; + + @Override + public void sampleFrom(ILodDataSource source) { + DhSectionPos pos = source.getSectionPos(); LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail); LodUtil.assertTrue(pos.overlaps(sectionPos)); - if (sparseSource.isEmpty) return; + if (source.isEmpty()) return; + if (source instanceof SparseDataSource) { + sampleFrom((SparseDataSource) source); + } else if (source instanceof FullDataSource) { + sampleFrom((FullDataSource) source); + } else { + LodUtil.assertNotReach(); + } + } + + private void sampleFrom(SparseDataSource sparseSource) { + DhSectionPos pos = sparseSource.getSectionPos(); isEmpty = false; - // Downsample needed DhLodPos basePos = sectionPos.getCorner(SPARSE_UNIT_DETAIL); DhLodPos dataPos = pos.getCorner(SPARSE_UNIT_DETAIL); int offsetX = dataPos.x-basePos.x; @@ -136,13 +150,10 @@ public class SparseDataSource implements ILodDataSource } } } - public void sampleFrom(FullDataSource fullSource) { + private void sampleFrom(FullDataSource fullSource) { DhSectionPos pos = fullSource.getSectionPos(); - LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail); - LodUtil.assertTrue(pos.overlaps(sectionPos)); - if (fullSource.isEmpty()) return; isEmpty = false; - // Downsample needed + DhLodPos basePos = sectionPos.getCorner(SPARSE_UNIT_DETAIL); DhLodPos dataPos = pos.getCorner(SPARSE_UNIT_DETAIL); int coveredChunks = pos.getWidth(SPARSE_UNIT_DETAIL).value; @@ -285,7 +296,7 @@ public class SparseDataSource implements ILodDataSource } } - public void applyToFullDataSource(FullDataSource dataSource) { + private void applyToFullDataSource(FullDataSource dataSource) { LodUtil.assertTrue(dataSource.getSectionPos().equals(sectionPos)); LodUtil.assertTrue(dataSource.getDataDetail() == getDataDetail()); for (int x = 0; x SparseDataSource.MAX_SECTION_DETAIL); + this.sectionPos = sectionPos; + isColumnNotEmpty = new BitSet(SECTION_SIZE*SECTION_SIZE); + } + + @Override + public DhSectionPos getSectionPos() { + return sectionPos; + } + @Override + public byte getDataDetail() { + return (byte) (sectionPos.sectionDetail-SECTION_SIZE_OFFSET); + } + + @Override + public byte getDataVersion() { + return LATEST_VERSION; + } + + @Override + public void update(ChunkSizedData data) { + LodUtil.assertTrue(sectionPos.getSectionBBoxPos().overlaps(data.getBBoxLodPos())); + + if (data.dataDetail == 0 && getDataDetail() >= 4) { + //FIXME: TEMPORARY + int chunkPerFull = 1 << (getDataDetail() - 4); + if (data.x % chunkPerFull != 0 || data.z % chunkPerFull != 0) return; + DhLodPos baseOffset = sectionPos.getCorner(getDataDetail()); + DhLodPos dataOffset = data.getBBoxLodPos().convertUpwardsTo(getDataDetail()); + int offsetX = dataOffset.x - baseOffset.x; + int offsetZ = dataOffset.z - baseOffset.z; + LodUtil.assertTrue(offsetX >= 0 && offsetX < SECTION_SIZE && offsetZ >= 0 && offsetZ < SECTION_SIZE); + isEmpty = false; + data.get(0,0).deepCopyTo(get(offsetX, offsetZ)); + } else { + LodUtil.assertNotReach(); + //TODO; + } + + } + + @Override + public boolean isEmpty() { + return isEmpty; + } + + public void markNotEmpty() { + isEmpty = false; + } + + @Override + public void saveData(IDhLevel level, DataMetaFile file, OutputStream dataStream) throws IOException { + DataOutputStream dos = new DataOutputStream(dataStream); // DO NOT CLOSE + { + dos.writeInt(getDataDetail()); + dos.writeInt(size); + dos.writeInt(level.getMinY()); + if (isEmpty) { + dos.writeInt(0x00000001); + return; + } + + // Is column not empty + dos.writeInt(0xFFFFFFFF); + byte[] bytes = isColumnNotEmpty.toByteArray(); + dos.writeInt(bytes.length); + dos.write(bytes); + + // Data array content + dos.writeInt(0xFFFFFFFF); + for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1)) + { + dos.writeByte(dataArrays[i].length); + if (dataArrays[i].length == 0) continue; + for (long l : dataArrays[i]) { + dos.writeLong(l); + } + } + + // Id mapping + dos.writeInt(0xFFFFFFFF); + mapping.serialize(dos); + dos.writeInt(0xFFFFFFFF); + } + } + + + public static SpottyDataSource loadData(DataMetaFile dataFile, InputStream dataStream, IDhLevel level) throws IOException { + DataInputStream dos = new DataInputStream(dataStream); // DO NOT CLOSE + { + int dataDetail = dos.readInt(); + if(dataDetail != dataFile.metaData.dataLevel) + throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); + int size = dos.readInt(); + if (size != SECTION_SIZE) + throw new IOException(LodUtil.formatLog( + "Section size mismatch: {} != {} (Currently only 1 section size is supported)", size, SECTION_SIZE)); + int minY = dos.readInt(); + if (minY != level.getMinY()) + LOGGER.warn("Data minY mismatch: {} != {}. Will ignore data's y level", minY, level.getMinY()); + int end = dos.readInt(); + // Data array length + if (end == 0x00000001) { + // Section is empty + return new SpottyDataSource(dataFile.pos); + } + + // Is column not empty + if (end != 0xFFFFFFFF) throw new IOException("invalid header end guard"); + int length = dos.readInt(); + + if (length < 0 || length > (SECTION_SIZE*SECTION_SIZE/8+64)*2) + throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})", + length, 1, SECTION_SIZE*SECTION_SIZE/8+63)); + byte[] bytes = dos.readNBytes(length); + BitSet isColumnNotEmpty = BitSet.valueOf(bytes); + + // Data array content + long[][] data = new long[SECTION_SIZE*SECTION_SIZE][]; + end = dos.readInt(); + if (end != 0xFFFFFFFF) throw new IOException("invalid spotty flag end guard"); + for (int i = isColumnNotEmpty.nextSetBit(0); i >= 0; i = isColumnNotEmpty.nextSetBit(i + 1)) + { + long[] array = new long[dos.readByte()]; + for (int j = 0; j < array.length; j++) { + array[j] = dos.readLong(); + } + data[i] = array; + } + + // Id mapping + end = dos.readInt(); + if (end != 0xFFFFFFFF) throw new IOException("invalid data content end guard"); + FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(new UnclosableInputStream(dos)); + end = dos.readInt(); + if (end != 0xFFFFFFFF) throw new IOException("invalid id mapping end guard"); + return new SpottyDataSource(dataFile.pos, mapping, isColumnNotEmpty, data); + } + } + + private SpottyDataSource(DhSectionPos pos, FullDataPointIdMap mapping, BitSet isColumnNotEmpty, long[][] data) { + super(mapping, data, SECTION_SIZE); + LodUtil.assertTrue(data.length == SECTION_SIZE*SECTION_SIZE); + this.sectionPos = pos; + this.isColumnNotEmpty = isColumnNotEmpty; + isEmpty = false; + } + + public static SpottyDataSource createEmpty(DhSectionPos pos) { + return new SpottyDataSource(pos); + } + + public static boolean neededForPosition(DhSectionPos posToWrite, DhSectionPos posToTest) { + if (!posToWrite.overlaps(posToTest)) return false; + if (posToTest.sectionDetail > posToWrite.sectionDetail) return false; + if (posToWrite.sectionDetail - posToTest.sectionDetail <= SECTION_SIZE_OFFSET) return true; + byte sectPerData = (byte) (1 << (posToWrite.sectionDetail - posToTest.sectionDetail - SECTION_SIZE_OFFSET)); + return posToTest.sectionX % sectPerData == 0 && posToTest.sectionZ % sectPerData == 0; + } + + @Override + public void sampleFrom(ILodDataSource source) { + DhSectionPos pos = source.getSectionPos(); + LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail); + LodUtil.assertTrue(pos.overlaps(sectionPos)); + if (source.isEmpty()) return; + if (source instanceof SparseDataSource) { + sampleFrom((SparseDataSource) source); + } else if (source instanceof FullDataSource) { + sampleFrom((FullDataSource) source); + } else { + LodUtil.assertNotReach(); + } + } + + private void sampleFrom(SparseDataSource sparseSource) { + DhSectionPos pos = sparseSource.getSectionPos(); + isEmpty = false; + + if (getDataDetail() > sectionPos.sectionDetail) { + DhLodPos basePos = sectionPos.getCorner(getDataDetail()); + DhLodPos dataPos = pos.getCorner(getDataDetail()); + int offsetX = dataPos.x - basePos.x; + int offsetZ = dataPos.z - basePos.z; + LodUtil.assertTrue(offsetX >= 0 && offsetX < SECTION_SIZE && offsetZ >= 0 && offsetZ < SECTION_SIZE); + int chunksPerData = 1 << (getDataDetail() - SparseDataSource.SPARSE_UNIT_DETAIL); + int dataSpan = sectionPos.getWidth(getDataDetail()).value; + + for (int ox = 0; ox < dataSpan; ox++) { + for (int oz = 0; oz < dataSpan; oz++) { + SingleFullArrayView column = sparseSource.tryGet( + ox * chunksPerData * sparseSource.dataPerChunk, + oz * chunksPerData * sparseSource.dataPerChunk); + if (column != null) { + column.deepCopyTo(get(offsetX + ox, offsetZ + oz)); + isColumnNotEmpty.set((offsetX + ox) * SECTION_SIZE + offsetZ + oz, true); + } + } + } + } else { + DhLodPos dataPos = pos.getSectionBBoxPos(); + int lowerSectionsPerData = sectionPos.getWidth(dataPos.detail).value; + if (dataPos.x % lowerSectionsPerData != 0 || dataPos.z % lowerSectionsPerData != 0) return; + + DhLodPos basePos = sectionPos.getCorner(getDataDetail()); + dataPos = dataPos.convertUpwardsTo(getDataDetail()); + int offsetX = dataPos.x - basePos.x; + int offsetZ = dataPos.z - basePos.z; + SingleFullArrayView column = sparseSource.tryGet(0, 0); + if (column != null) { + column.deepCopyTo(get(offsetX, offsetZ)); + isColumnNotEmpty.set(offsetX * SECTION_SIZE + offsetZ, true); + } + } + } + + private void sampleFrom(FullDataSource fullSource) { + DhSectionPos pos = fullSource.getSectionPos(); + isEmpty = false; + downsampleFrom(fullSource); + + if (getDataDetail() > sectionPos.sectionDetail) { + DhLodPos basePos = sectionPos.getCorner(getDataDetail()); + DhLodPos dataPos = pos.getCorner(getDataDetail()); + int offsetX = dataPos.x - basePos.x; + int offsetZ = dataPos.z - basePos.z; + int dataSpan = sectionPos.getWidth(getDataDetail()).value; + for (int ox = 0; ox < dataSpan; ox++) { + for (int oz = 0; oz < dataSpan; oz++) { + isColumnNotEmpty.set((offsetX + ox) * SECTION_SIZE + offsetZ + oz, true); + } + } + } else { + DhLodPos dataPos = pos.getSectionBBoxPos(); + int lowerSectionsPerData = sectionPos.getWidth(dataPos.detail).value; + if (dataPos.x % lowerSectionsPerData != 0 || dataPos.z % lowerSectionsPerData != 0) return; + DhLodPos basePos = sectionPos.getCorner(getDataDetail()); + dataPos = dataPos.convertUpwardsTo(getDataDetail()); + int offsetX = dataPos.x - basePos.x; + int offsetZ = dataPos.z - basePos.z; + isColumnNotEmpty.set(offsetX * SECTION_SIZE + offsetZ, true); + } + + } + + @Override + public ILodDataSource trySelfPromote() { + if (isEmpty) return this; + if (isColumnNotEmpty.cardinality() != SECTION_SIZE * SECTION_SIZE) return this; + return new FullDataSource(sectionPos, mapping, dataArrays); + } + + @Override + public SingleFullArrayView tryGet(int x, int z) { + return isColumnNotEmpty.get(x * SECTION_SIZE + z) ? get(x, z) : null; + } +} diff --git a/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java b/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java index 83e1c4905..0456ff00a 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java @@ -1,5 +1,6 @@ package com.seibel.lod.core.datatype.transform; +import com.seibel.lod.core.datatype.IIncompleteDataSource; import com.seibel.lod.core.datatype.ILodRenderSource; import com.seibel.lod.core.datatype.column.accessor.ColumnFormat; import com.seibel.lod.core.datatype.column.ColumnRenderSource; @@ -66,12 +67,12 @@ public class FullToColumnTransformer { return columnSource; } - public static ILodRenderSource transformSparseDataToColumnData(IDhClientLevel level, SparseDataSource data) { + public static ILodRenderSource transformIncompleteDataToColumnData(IDhClientLevel level, IIncompleteDataSource data) { final DhSectionPos pos = data.getSectionPos(); final byte dataDetail = data.getDataDetail(); final int vertSize = Config.Client.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(data.getDataDetail()); final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY()); - if (data.isEmpty) return columnSource; + if (data.isEmpty()) return columnSource; columnSource.markNotEmpty(); if (dataDetail == columnSource.getDataDetail()) { diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java index 931f75822..34142d9fe 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java @@ -1,10 +1,12 @@ package com.seibel.lod.core.file.datafile; import com.google.common.collect.HashMultimap; +import com.seibel.lod.core.datatype.IIncompleteDataSource; import com.seibel.lod.core.datatype.ILodDataSource; import com.seibel.lod.core.datatype.full.ChunkSizedData; import com.seibel.lod.core.datatype.full.FullDataSource; import com.seibel.lod.core.datatype.full.SparseDataSource; +import com.seibel.lod.core.datatype.full.SpottyDataSource; import com.seibel.lod.core.file.MetaFile; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.pos.DhLodPos; @@ -255,10 +257,17 @@ public class DataFileHandler implements IDataSourceProvider { } @Override - public long getLatestCacheVersion(DhSectionPos sectionPos) { + public long getCacheVersion(DhSectionPos sectionPos) { DataMetaFile file = files.get(sectionPos); if (file == null) return 0; - return file.getDataVersion(); + return file.getCacheVersion(); + } + + @Override + public boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion) { + DataMetaFile file = files.get(sectionPos); + if (file == null) return cacheVersion >= 0; + return file.isCacheVersionValid(cacheVersion); } @Override @@ -270,27 +279,25 @@ public class DataFileHandler implements IDataSourceProvider { LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty()); if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) { // None exist. - SparseDataSource dataSource = SparseDataSource.createEmpty(pos); - return CompletableFuture.completedFuture(dataSource); + IIncompleteDataSource incompleteDataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ? + SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos); + return CompletableFuture.completedFuture(incompleteDataSource); } else { - for (DhSectionPos missingPos : missing) { DataMetaFile newfile = atomicGetOrMakeFile(missingPos); if (newfile != null) existFiles.add(newfile); } final ArrayList> futures = new ArrayList<>(existFiles.size()); - final SparseDataSource dataSource = SparseDataSource.createEmpty(pos); + final IIncompleteDataSource dataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ? + SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos); for (DataMetaFile f : existFiles) { futures.add(f.loadOrGetCached() .exceptionally((ex) -> null) .thenAccept((data) -> { if (data != null) { - if (data instanceof SparseDataSource) - dataSource.sampleFrom((SparseDataSource) data); - else if (data instanceof FullDataSource) - dataSource.sampleFrom((FullDataSource) data); - else LodUtil.assertNotReach(); + LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); + dataSource.sampleFrom(data); } }) ); @@ -305,8 +312,8 @@ public class DataFileHandler implements IDataSourceProvider { Consumer onUpdated, Function updater) { boolean changed = updater.apply(source); if (changed) metaData.dataVersion.incrementAndGet(); - if (source instanceof SparseDataSource) { - ILodDataSource newSource = ((SparseDataSource) source).trySelfPromote(); + if (source instanceof IIncompleteDataSource) { + ILodDataSource newSource = ((IIncompleteDataSource) source).trySelfPromote(); changed |= newSource != source; source = newSource; } @@ -314,12 +321,13 @@ public class DataFileHandler implements IDataSourceProvider { return source; } @Override - public CompletableFuture onDataFileRefresh(ILodDataSource source, Function updater, Consumer onUpdated) { + public CompletableFuture onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function updater, Consumer onUpdated) { return CompletableFuture.supplyAsync(() -> { ILodDataSource sourceLocal = source; boolean changed = updater.apply(sourceLocal); - if (sourceLocal instanceof SparseDataSource) { - ILodDataSource newSource = ((SparseDataSource) sourceLocal).trySelfPromote(); + if (changed) metaData.dataVersion.incrementAndGet(); + if (sourceLocal instanceof IIncompleteDataSource) { + ILodDataSource newSource = ((IIncompleteDataSource) sourceLocal).trySelfPromote(); changed |= newSource != sourceLocal; sourceLocal = newSource; } diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java index 70016b436..e1411cde7 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java @@ -17,6 +17,7 @@ import com.seibel.lod.core.file.MetaFile; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.util.AtomicsUtil; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; @@ -48,7 +49,7 @@ public class DataMetaFile extends MetaFile GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue(); // =========================== - private final AtomicBoolean inCacheWriteAccessAsserter = new AtomicBoolean(false); + private AtomicReference> inCacheWriteAccessFuture = new AtomicReference<>(null); // ===Object lifetime stuff=== private static final ReferenceQueue lifeCycleDebugQueue = new ReferenceQueue<>(); @@ -105,16 +106,29 @@ public class DataMetaFile extends MetaFile } } - public long getDataVersion() { + public long getCacheVersion() { debugCheck(); MetaData getData = metaData; return getData == null ? 0 : metaData.dataVersion.get(); } + public boolean isCacheVersionValid(long cacheVersion) { + debugCheck(); + boolean noWrite = writeQueue.get().queue.isEmpty(); + if (!noWrite) { + return false; + } else { + MetaData getData = metaData; + //NOTE: Do this instead of direct compare so values that wrapped around still works correctly. + return (getData == null ? 0 : metaData.dataVersion.get()) - cacheVersion <= 0; + } + } + public void addToWriteQueue(ChunkSizedData datatype) { debugCheck(); DhLodPos chunkPos = new DhLodPos((byte) (datatype.dataDetail + 4), datatype.x, datatype.z); LodUtil.assertTrue(pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos {} doesn't overlap with section {}", chunkPos, pos); + //LOGGER.info("Write Chunk {} to file {}", chunkPos, pos); GuardedMultiAppendQueue queue = writeQueue.get(); // Using read lock is OK, because the queue's underlying data structure is thread-safe. @@ -175,7 +189,7 @@ public class DataMetaFile extends MetaFile throw new CompletionException(e); } // Apply the write queue - LodUtil.assertTrue(!inCacheWriteAccessAsserter.get(),"No one should be writing to the cache while we are in the process of " + + LodUtil.assertTrue(inCacheWriteAccessFuture.get() == null,"No one should be writing to the cache while we are in the process of " + "loading one into the cache! Is this a deadlock?"); data = handler.onDataFileLoaded(data, metaData, this::saveChanges, this::applyWriteQueue); // Finally, return the data. @@ -207,7 +221,7 @@ public class DataMetaFile extends MetaFile // "unchecked": Suppress casting of CompletableFuture to CompletableFuture // "PointlessBooleanExpression": Suppress explicit (boolean == false) check for more understandable CAS operation code. - @SuppressWarnings({"unchecked", "PointlessBooleanExpression"}) + @SuppressWarnings({"unchecked"}) private CompletableFuture _readCached(Object obj) { // Has file cached in RAM and not freed yet. if ((obj instanceof SoftReference)) { @@ -219,23 +233,33 @@ public class DataMetaFile extends MetaFile // that will be applying the changes to the cache. if (!isEmpty) { // Do a CAS on inCacheWriteLock to ensure that we are the only thread that is writing to the cache, - // or if we fail, then that means someone else is already doing it, and we can just continue. - // FIXME: Should we return a future that waits for the write to be done for CAS fail? Or should we just return the - // cached data that doesn't have all writes done immediately? - // The latter give us immediate access to the data, but we need to ensure concurrent reads and - // writes doesn't cause unexpected behavior down the line. - // For now, I'll go for the latter option and just hope nothing goes wrong... - if (inCacheWriteAccessAsserter.getAndSet(true) == false) { + // or if we fail, then that means someone else is already doing it, and we can just return the future + CompletableFuture future = new CompletableFuture<>(); + CompletableFuture cas = AtomicsUtil.compareAndExchange(inCacheWriteAccessFuture, null, future); + if (cas == null) { try { - return handler.onDataFileRefresh((ILodDataSource) inner, this::applyWriteQueue, this::saveChanges); + data.set(future); + handler.onDataFileRefresh((ILodDataSource) inner, metaData, this::applyWriteQueue, this::saveChanges).handle((v, e) -> { + if (e != null) { + LOGGER.error("Error refreshing data {}: ", pos, e); + future.complete(null); + data.set(null); + } else { + future.complete(v); + new DataObjTracker(v); + data.set(new SoftReference<>(v)); + } + inCacheWriteAccessFuture.set(null); + return v; + }); + return future; } catch (Exception e) { - LOGGER.error("Error while applying changes to LodDataSource at {}: ", pos, e); - } finally { - inCacheWriteAccessAsserter.set(false); + LOGGER.error("Error while doing refreshes to LodDataSource at {}: ", pos, e); + return CompletableFuture.completedFuture((ILodDataSource) inner); } } else { - // or, return the cached data. FIXME: See above. - return CompletableFuture.completedFuture((ILodDataSource) inner); + // or, return the future that will be completed when the write is done. + return cas; } } else { // or, return the cached data. @@ -270,6 +294,7 @@ public class DataMetaFile extends MetaFile if (path.exists()) if (!path.delete()) LOGGER.warn("Failed to delete data file at {}", path); doesFileExist = false; } else { + LOGGER.info("Saving data file of {}", data.getSectionPos()); try { // Write/Update data LodUtil.assertTrue(metaData != null); diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/GeneratedDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/datafile/GeneratedDataFileHandler.java index 784fd2700..96766b48a 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/GeneratedDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/GeneratedDataFileHandler.java @@ -1,9 +1,10 @@ package com.seibel.lod.core.file.datafile; +import com.seibel.lod.core.datatype.IIncompleteDataSource; import com.seibel.lod.core.datatype.ILodDataSource; import com.seibel.lod.core.datatype.full.ChunkSizedData; -import com.seibel.lod.core.datatype.full.FullDataSource; import com.seibel.lod.core.datatype.full.SparseDataSource; +import com.seibel.lod.core.datatype.full.SpottyDataSource; import com.seibel.lod.core.generation.GenerationQueue; import com.seibel.lod.core.level.IDhServerLevel; import com.seibel.lod.core.pos.DhSectionPos; @@ -99,7 +100,8 @@ public class GeneratedDataFileHandler extends DataFileHandler { LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty()); if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) { // None exist. - SparseDataSource dataSource = SparseDataSource.createEmpty(pos); + IIncompleteDataSource dataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ? + SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos); GenerationQueue getQueue = queue.get(); GenTask task = new GenTask(pos, new WeakReference<>(dataSource)); genQueue.put(dataSource, task); @@ -126,18 +128,17 @@ public class GeneratedDataFileHandler extends DataFileHandler { if (newfile != null) existFiles.add(newfile); } final ArrayList> futures = new ArrayList<>(existFiles.size()); - final SparseDataSource dataSource = SparseDataSource.createEmpty(pos); + final IIncompleteDataSource dataSource = pos.sectionDetail <= SparseDataSource.MAX_SECTION_DETAIL ? + SparseDataSource.createEmpty(pos) : SpottyDataSource.createEmpty(pos); + LOGGER.debug("Creating {} from sampling {} files: {}", pos, existFiles.size(), existFiles); for (DataMetaFile f : existFiles) { futures.add(f.loadOrGetCached() .exceptionally((ex) -> null) .thenAccept((data) -> { if (data != null) { - if (data instanceof SparseDataSource) - dataSource.sampleFrom((SparseDataSource) data); - else if (data instanceof FullDataSource) - dataSource.sampleFrom((FullDataSource) data); - else LodUtil.assertNotReach(); + LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); + dataSource.sampleFrom(data); } }) ); diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java b/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java index 875ec53e6..396531f1f 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java @@ -19,11 +19,12 @@ public interface IDataSourceProvider extends AutoCloseable { void write(DhSectionPos sectionPos, ChunkSizedData chunkData); CompletableFuture flushAndSave(); - long getLatestCacheVersion(DhSectionPos sectionPos); + long getCacheVersion(DhSectionPos sectionPos); + boolean isCacheVersionValid(DhSectionPos sectionPos, long cacheVersion); CompletableFuture onCreateDataFile(DataMetaFile file); ILodDataSource onDataFileLoaded(ILodDataSource source, MetaFile.MetaData metaData, Consumer onUpdated, Function updater); - CompletableFuture onDataFileRefresh(ILodDataSource source, Function updater, Consumer onUpdated); + CompletableFuture onDataFileRefresh(ILodDataSource source, MetaFile.MetaData metaData, Function updater, Consumer onUpdated); File computeDataFilePath(DhSectionPos pos); Executor getIOExecutor(); diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java index 3de8c18b0..9ee39089b 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java @@ -17,6 +17,7 @@ import com.seibel.lod.core.config.Config; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; +import org.lwjgl.system.CallbackI; import java.io.File; import java.io.IOException; @@ -204,33 +205,31 @@ public class RenderFileHandler implements IRenderSourceProvider { if (cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null) return; final WeakReference dataRef = new WeakReference<>(data); CompletableFuture dataFuture = dataSourceProvider.read(data.getSectionPos()); - final long version = dataSourceProvider.getLatestCacheVersion(data.getSectionPos()); - DataRenderTransformer.asyncTransformDataSource( - dataFuture.thenApply((d) -> { - if (dataRef.get() == null) throw new UncheckedInterruptedException(); - LodUtil.assertTrue(d != null); - return d; - }).exceptionally((ex) -> { - if (ex != null) - LOGGER.error("Uncaught exception when getting data for updateCache()", ex); - return null; - }) - , level) - .thenAccept((newData) -> write(dataRef.get(), file, newData, version)) - .exceptionally((ex) -> { - if (!UncheckedInterruptedException.isThrowableInterruption(ex)) - LOGGER.error("Exception when updating render file using data source: ", ex); - return null; - }).thenRun(() -> cacheRecreationGuards.remove(file.pos)); + dataFuture = dataFuture.thenApply((d) -> { + if (dataRef.get() == null) throw new UncheckedInterruptedException(); + LodUtil.assertTrue(d != null); + return d; + }).exceptionally((ex) -> { + if (ex != null) + LOGGER.error("Uncaught exception when getting data for updateCache()", ex); + return null; + }); + + LOGGER.info("Recreating cache for {}", data.getSectionPos()); + DataRenderTransformer.asyncTransformDataSource(dataFuture , level) + .thenAccept((newData) -> write(dataRef.get(), file, newData, dataSourceProvider.getCacheVersion(data.getSectionPos()))) + .exceptionally((ex) -> { + if (!UncheckedInterruptedException.isThrowableInterruption(ex)) + LOGGER.error("Exception when updating render file using data source: ", ex); + return null; + }).thenRun(() -> cacheRecreationGuards.remove(file.pos)); } public ILodRenderSource onRenderFileLoaded(ILodRenderSource data, RenderMetaFile file) { - long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos); - //NOTE: Do this instead of direct compare so values that wrapped around still works correctly. - if (newCacheVersion - file.metaData.dataVersion.get() <= 0) - return data; - updateCache(data, file); + if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) { + updateCache(data, file); + } return data; } @@ -253,10 +252,9 @@ public class RenderFileHandler implements IRenderSourceProvider { } public void onReadRenderSourceFromCache(RenderMetaFile file, ILodRenderSource data) { - long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos); - //NOTE: Do this instead of direct compare so values that wrapped around still works correctly. - if (newCacheVersion - file.metaData.dataVersion.get() > 0) + if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) { updateCache(data, file); + } } public boolean refreshRenderSource(ILodRenderSource source) { @@ -268,12 +266,11 @@ public class RenderFileHandler implements IRenderSourceProvider { } LodUtil.assertTrue(file != null); LodUtil.assertTrue(file.metaData != null); - long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos); - //NOTE: Do this instead of direct compare so values that wrapped around still works correctly. - if (newCacheVersion - file.metaData.dataVersion.get() <= 0) - return false; - updateCache(source, file); - return true; + if (!dataSourceProvider.isCacheVersionValid(file.pos, file.metaData.dataVersion.get())) { + updateCache(source, file); + return true; + } + return false; } }