diff --git a/core/src/main/java/com/seibel/lod/core/a7/datatype/LodDataSource.java b/core/src/main/java/com/seibel/lod/core/a7/datatype/LodDataSource.java index 6d4d61e18..d9e084f07 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/datatype/LodDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/a7/datatype/LodDataSource.java @@ -14,7 +14,6 @@ import java.io.OutputStream; public interface LodDataSource { DhSectionPos getSectionPos(); byte getDataDetail(); - void setLocalVersion(int localVer); byte getDataVersion(); void update(ChunkSizedData data); diff --git a/core/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java b/core/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java index 968e98266..e75a93149 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java @@ -13,12 +13,12 @@ import java.util.concurrent.atomic.AtomicReference; public interface LodRenderSource { DhSectionPos getSectionPos(); + byte getDataDetail(); void enableRender(IClientLevel level, LodQuadTree quadTree); void disableRender(); boolean isRenderReady(); void dispose(); // notify the container that the parent lodSection is now disposed (can be in loaded or unloaded state) - byte getDetailOffset(); /** @@ -32,7 +32,9 @@ public interface LodRenderSource { void saveRender(IClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException; + @Deprecated void write(ChunkSizedData chunkData); + @Deprecated void flushWrites(IClientLevel level); byte getRenderVersion(); @@ -41,4 +43,7 @@ public interface LodRenderSource { * Whether this object is still valid. If not, a new one should be created. */ boolean isValid(); + + // Only override the data that has not been written directly using write(), and skip those that are empty + void weakWrite(LodRenderSource source); } diff --git a/core/src/main/java/com/seibel/lod/core/a7/datatype/PlaceHolderRenderSource.java b/core/src/main/java/com/seibel/lod/core/a7/datatype/PlaceHolderRenderSource.java index 7f8dea0ed..3f9b216ad 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/datatype/PlaceHolderRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/a7/datatype/PlaceHolderRenderSource.java @@ -23,6 +23,11 @@ public class PlaceHolderRenderSource implements LodRenderSource { return pos; } + @Override + public byte getDataDetail() { + return 0; + } + @Override public void enableRender(IClientLevel level, LodQuadTree quadTree) { } @@ -37,10 +42,6 @@ public class PlaceHolderRenderSource implements LodRenderSource { @Override public void dispose() {} @Override - public byte getDetailOffset() { - return 0; - } - @Override public boolean trySwapRenderBuffer(LodQuadTree quadTree, AtomicReference referenceSlotsOpaque, AtomicReference referenceSlotsTransparent) { return false; } @@ -68,5 +69,10 @@ public class PlaceHolderRenderSource implements LodRenderSource { return isValid; } + @Override + public void weakWrite(LodRenderSource source) { + + } + } diff --git a/core/src/main/java/com/seibel/lod/core/a7/datatype/RenderSourceLoader.java b/core/src/main/java/com/seibel/lod/core/a7/datatype/RenderSourceLoader.java index cfb17ffc6..a68ab8a35 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/datatype/RenderSourceLoader.java +++ b/core/src/main/java/com/seibel/lod/core/a7/datatype/RenderSourceLoader.java @@ -2,6 +2,7 @@ package com.seibel.lod.core.a7.datatype; import com.google.common.collect.HashMultimap; import com.seibel.lod.core.a7.level.IClientLevel; +import com.seibel.lod.core.a7.level.ILevel; import com.seibel.lod.core.a7.save.io.render.RenderMetaFile; import java.io.IOException; @@ -53,7 +54,7 @@ public abstract class RenderSourceLoader { } // Can return null as meaning the file is out of date or something - public abstract LodRenderSource loadRender(RenderMetaFile renderFile, InputStream data, IClientLevel level) throws IOException; + public abstract LodRenderSource loadRender(RenderMetaFile renderFile, InputStream data, ILevel level) throws IOException; public abstract LodRenderSource createRender(LodDataSource dataSource, IClientLevel level); diff --git a/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java b/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java index 115c736c8..e30072e8f 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java +++ b/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java @@ -2,11 +2,14 @@ package com.seibel.lod.core.a7.datatype.column; import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.full.FullDataSource; +import com.seibel.lod.core.a7.datatype.full.SparseDataSource; import com.seibel.lod.core.a7.datatype.transform.FullToColumnTransformer; import com.seibel.lod.core.a7.level.IClientLevel; import com.seibel.lod.core.a7.datatype.LodRenderSource; import com.seibel.lod.core.a7.datatype.RenderSourceLoader; +import com.seibel.lod.core.a7.level.ILevel; import com.seibel.lod.core.a7.save.io.render.RenderMetaFile; +import com.seibel.lod.core.util.LodUtil; import java.io.DataInputStream; import java.io.IOException; @@ -18,12 +21,12 @@ public class ColumnRenderLoader extends RenderSourceLoader { } @Override - public LodRenderSource loadRender(RenderMetaFile dataFile, InputStream data, IClientLevel level) throws IOException { + public LodRenderSource loadRender(RenderMetaFile dataFile, InputStream data, ILevel level) throws IOException { try ( //TODO: Add decompressor here DataInputStream dis = new DataInputStream(data); ) { - return new ColumnRenderSource(dataFile.pos, dis, dataFile.loaderVersion, level); + return new ColumnRenderSource(dataFile.pos, dis, dataFile.metaData.loaderVersion, level); } } @@ -31,7 +34,10 @@ public class ColumnRenderLoader extends RenderSourceLoader { public LodRenderSource createRender(LodDataSource dataSource, IClientLevel level) { if (dataSource instanceof FullDataSource) { return FullToColumnTransformer.transformFullDataToColumnData(level, (FullDataSource) dataSource); + } else if (dataSource instanceof SparseDataSource) { + return FullToColumnTransformer.transformSparseDataToColumnData(level, (SparseDataSource) dataSource); } + LodUtil.assertNotReach(); return null; } diff --git a/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java b/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java index 819a88f0e..d0700319f 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java @@ -210,6 +210,8 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype { for (int j = 0; j < verticalSize; j++) { long current = dataContainer[i * verticalSize + j]; + if (ColumnFormat.doesItExist(current)) + current = ColumnFormat.overrideGenerationMode(current, (byte) 1); output.writeLong(Long.reverseBytes(current)); } if (!ColumnFormat.doesItExist(dataContainer[i])) @@ -402,4 +404,26 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype { public boolean isValid() { return true; } + + @Override + public void weakWrite(LodRenderSource source) { + LodUtil.assertTrue(source instanceof ColumnRenderSource); + ColumnRenderSource src = (ColumnRenderSource) source; + + LodUtil.assertTrue(src.sectionPos.equals(sectionPos)); + LodUtil.assertTrue(src.verticalSize == verticalSize); + + if (src.isEmpty) return; + isEmpty = false; + + for (int i=0; i=0 && offsetX < chunks && offsetZ >=0 && offsetZ < chunks); + + for (int ox = 0; ox < sparseSource.chunks; ox++) { + for (int oz = 0; oz < sparseSource.chunks; oz++) { + FullArrayView sourceChunk = sparseSource.sparseData[ox*sparseSource.chunks + oz]; + if (sourceChunk != null) { + FullArrayView buff = new FullArrayView(mapping, new long[dataPerChunk * dataPerChunk][], dataPerChunk); + buff.downsampleFrom(sourceChunk); + sparseData[(ox+offsetX)* chunks + (oz+offsetZ)] = buff; + } + } + } + } + public void sampleFrom(FullDataSource fullSource) { + DhSectionPos pos = fullSource.getSectionPos(); + LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail); + LodUtil.assertTrue(pos.overlaps(sectionPos)); + if (fullSource.isEmpty) return; + // Downsample needed + DhLodPos basePos = sectionPos.getCorner(SPARSE_UNIT_DETAIL); + DhLodPos dataPos = pos.getCorner(SPARSE_UNIT_DETAIL); + int coveredChunks = pos.getWidth(SPARSE_UNIT_DETAIL).value; + int sourceDataPerChunk = SPARSE_UNIT_SIZE >>> fullSource.getDataDetail(); + LodUtil.assertTrue(coveredChunks*sourceDataPerChunk == FullDataSource.SECTION_SIZE); + int offsetX = dataPos.x-basePos.x; + int offsetZ = dataPos.z-basePos.z; + LodUtil.assertTrue(offsetX >=0 && offsetX < chunks && offsetZ >=0 && offsetZ < chunks); + + for (int ox = 0; ox < coveredChunks; ox++) { + for (int oz = 0; oz < coveredChunks; oz++) { + FullArrayView sourceChunk = fullSource.subView(sourceDataPerChunk, ox*sourceDataPerChunk, oz*sourceDataPerChunk); + FullArrayView buff = new FullArrayView(mapping, new long[dataPerChunk * dataPerChunk][], dataPerChunk); + buff.downsampleFrom(sourceChunk); + sparseData[(ox+offsetX)* chunks + (oz+offsetZ)] = buff; + } + } + } + @Override public void saveData(ILevel level, DataMetaFile file, OutputStream dataStream) throws IOException { try (DataOutputStream dos = new DataOutputStream(dataStream)) { @@ -160,8 +207,8 @@ public class SparseDataSource implements LodDataSource { LodUtil.assertTrue(dataFile.pos.sectionDetail <= MAX_SECTION_DETAIL); try (DataInputStream dos = new DataInputStream(dataStream)) { int dataDetail = dos.readShort(); - if(dataDetail != dataFile.dataLevel) - throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.dataLevel)); + if(dataDetail != dataFile.metaData.dataLevel) + throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel)); int sparseDetail = dos.readShort(); if (sparseDetail != SPARSE_UNIT_DETAIL) throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}", @@ -204,8 +251,8 @@ public class SparseDataSource implements LodDataSource { for (int i = set.nextSetBit(0); i >= 0 && i < dataChunks.length; i = set.nextSetBit(i + 1)) { long[][] dataColumns = new long[dataPerChunk*dataPerChunk][]; dataChunks[i] = dataColumns; - for (int j = 0; j < dataColumns.length; j++) { - dataColumns[i] = new long[dos.readByte()]; + for (int i2 = 0; i2 < dataColumns.length; i2++) { + dataColumns[i2] = new long[dos.readByte()]; } for (int k = 0; k < dataColumns.length; k++) { if (dataColumns[k].length == 0) continue; @@ -240,9 +287,43 @@ public class SparseDataSource implements LodDataSource { FullArrayView array = sparseData[x*chunks+z]; if (array == null) continue; // Otherwise, apply data to dataSource + dataSource.isEmpty = false; FullArrayView view = dataSource.subView(dataPerChunk, x*dataPerChunk, z*dataPerChunk); array.shadowCopyTo(view); } } } + + + public LodDataSource promote(LodDataSource generatedData) { + if (!(generatedData instanceof FullDataSource) && !(generatedData instanceof SparseDataSource)) + throw new UnsupportedOperationException("Requires FullDataSource for the promotion!"); + if (generatedData instanceof FullDataSource) { + applyToFullDataSource((FullDataSource) generatedData); + return generatedData; + } else { + LodUtil.assertToDo(); //TODO + return null; + } + } + + public LodDataSource trySelfPromote() { + if (isEmpty) return this; + for (FullArrayView array : sparseData) { + if (array == null) return this; + } + FullDataSource newSource = FullDataSource.createEmpty(sectionPos); + applyToFullDataSource(newSource); + return newSource; + } + + // Return null if doesn't exist + public SingleFullArrayView tryGet(int x, int z) { + LodUtil.assertTrue(x>=0 && x=0 && z size && source.size % size == 0); + int dataPerUnit = source.size / size; + for (int ox = 0; ox < size; ox++) { + for (int oz = 0; oz < size; oz++) { + SingleFullArrayView column = get(ox, oz); + column.downsampleFrom(source.subView(dataPerUnit, ox * dataPerUnit, oz * dataPerUnit)); + } + } + } } diff --git a/core/src/main/java/com/seibel/lod/core/a7/datatype/transform/FullToColumnTransformer.java b/core/src/main/java/com/seibel/lod/core/a7/datatype/transform/FullToColumnTransformer.java index 90b56f5d0..58ed2921b 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/datatype/transform/FullToColumnTransformer.java +++ b/core/src/main/java/com/seibel/lod/core/a7/datatype/transform/FullToColumnTransformer.java @@ -1,13 +1,11 @@ package com.seibel.lod.core.a7.datatype.transform; +import com.seibel.lod.core.a7.datatype.LodRenderSource; import com.seibel.lod.core.a7.datatype.column.accessor.ColumnFormat; import com.seibel.lod.core.a7.datatype.column.ColumnRenderSource; import com.seibel.lod.core.a7.datatype.column.accessor.ColumnArrayView; import com.seibel.lod.core.a7.datatype.column.accessor.ColumnQuadView; -import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; -import com.seibel.lod.core.a7.datatype.full.FullDataSource; -import com.seibel.lod.core.a7.datatype.full.FullFormat; -import com.seibel.lod.core.a7.datatype.full.IdBiomeBlockStateMap; +import com.seibel.lod.core.a7.datatype.full.*; import com.seibel.lod.core.a7.datatype.full.accessor.SingleFullArrayView; import com.seibel.lod.core.a7.level.IClientLevel; import com.seibel.lod.core.a7.pos.DhSectionPos; @@ -43,7 +41,7 @@ public class FullToColumnTransformer { for (int z = 0; z < pos.getWidth(dataDetail).value; z++) { ColumnArrayView columnArrayView = columnSource.getVerticalDataView(x, z); SingleFullArrayView fullArrayView = data.get(x, z); - convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView); + convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView, 1); if (fullArrayView.doesItExist()) LodUtil.assertTrue(columnSource.doesItExist(x, z)); } } @@ -67,6 +65,33 @@ public class FullToColumnTransformer { return columnSource; } + public static LodRenderSource transformSparseDataToColumnData(IClientLevel level, SparseDataSource data) { + final DhSectionPos pos = data.getSectionPos(); + final byte dataDetail = data.getDataDetail(); + final int vertSize = Config.Client.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(data.getDataDetail()); + final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY()); + if (data.isEmpty) return columnSource; + columnSource.isEmpty = false; + + if (dataDetail == columnSource.getDataDetail()) { + int baseX = pos.getCorner().getCorner().x; + int baseZ = pos.getCorner().getCorner().z; + for (int x = 0; x < pos.getWidth(dataDetail).value; x++) { + for (int z = 0; z < pos.getWidth(dataDetail).value; z++) { + SingleFullArrayView fullArrayView = data.tryGet(x, z); + if (fullArrayView == null) continue; + ColumnArrayView columnArrayView = columnSource.getVerticalDataView(x, z); + convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView, 1); + if (fullArrayView.doesItExist()) LodUtil.assertTrue(columnSource.doesItExist(x, z)); + } + } + } else { + throw new UnsupportedOperationException("To be implemented"); + //FIXME: Implement different size creation of renderData + } + return columnSource; + } + public static void writeFullDataChunkToColumnData(ColumnRenderSource render, IClientLevel level, ChunkSizedData data) { if (data.dataDetail != 0) throw new UnsupportedOperationException("To be implemented"); @@ -87,7 +112,7 @@ public class FullToColumnTransformer { SingleFullArrayView fullArrayView = data.get(x, z); convertColumnData(level, blockX + perRenderWidth * (renderOffsetX+x), blockZ + perRenderWidth * (renderOffsetZ+z), - columnArrayView, fullArrayView); + columnArrayView, fullArrayView, 2); if (fullArrayView.doesItExist()) LodUtil.assertTrue(render.doesItExist(renderOffsetX + x, renderOffsetZ + z)); } } @@ -108,7 +133,7 @@ public class FullToColumnTransformer { SingleFullArrayView fullArrayView = data.get(x*dataPerRender+ox, z*dataPerRender+oz); convertColumnData(level, blockX + perRenderWidth * (renderOffsetX+x) + perDataWidth * ox, blockZ + perRenderWidth * (renderOffsetZ+z) + perDataWidth * oz, - columnArrayView, fullArrayView); + columnArrayView, fullArrayView, 2); } } ColumnArrayView downSampledArrayView = render.getVerticalDataView(renderOffsetX + x, renderOffsetZ + z); @@ -118,19 +143,17 @@ public class FullToColumnTransformer { } } - private static void convertColumnData(IClientLevel level, int blockX, int blockZ, ColumnArrayView columnArrayView, SingleFullArrayView fullArrayView) { + private static void convertColumnData(IClientLevel level, int blockX, int blockZ, ColumnArrayView columnArrayView, SingleFullArrayView fullArrayView, int genMode) { if (!fullArrayView.doesItExist()) return; - // TODO: Set gen mode - int genModeValue = 1; int dataTotalLength = fullArrayView.getSingleLength(); if (dataTotalLength == 0) return; if (dataTotalLength > columnArrayView.verticalSize()) { ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength); - iterateAndConvert(level, blockX, blockZ, genModeValue, totalColumnData, fullArrayView); + iterateAndConvert(level, blockX, blockZ, genMode, totalColumnData, fullArrayView); columnArrayView.changeVerticalSizeFrom(totalColumnData); } else { - iterateAndConvert(level, blockX, blockZ, genModeValue, columnArrayView, fullArrayView); //Directly use the arrayView since it fits. + iterateAndConvert(level, blockX, blockZ, genMode, columnArrayView, fullArrayView); //Directly use the arrayView since it fits. } } diff --git a/core/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java b/core/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java index a8a95e5f7..a246cb102 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java +++ b/core/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java @@ -1,213 +1,398 @@ package com.seibel.lod.core.a7.generation; -import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; -import com.seibel.lod.core.a7.datatype.full.FullDataSource; import com.seibel.lod.core.a7.pos.DhBlockPos2D; import com.seibel.lod.core.a7.pos.DhLodPos; -import com.seibel.lod.core.a7.pos.DhSectionPos; -import com.seibel.lod.core.a7.util.ConcurrentQuadCombinableProviderTree; -import com.seibel.lod.core.a7.util.UncheckedInterruptedException; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.objects.DHChunkPos; import com.seibel.lod.core.util.LodUtil; -import com.seibel.lod.core.util.gridList.ArrayGridList; import org.apache.logging.log4j.Logger; +import java.io.Closeable; import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +public class GenerationQueue implements Closeable { + public static final int SHUTDOWN_TIMEOUT_SEC = 10; + -public class GenerationQueue implements AutoCloseable { - final ConcurrentQuadCombinableProviderTree cqcpTree = new ConcurrentQuadCombinableProviderTree<>(); - IGenerator generator = null; //FIXME: This is volatile and need atomic control private final Logger logger = DhLoggerBuilder.getLogger(); - private final ConcurrentHashMap> taskMap = new ConcurrentHashMap<>(); - private final AtomicReference>> inProgress = new AtomicReference<>(null); + public static abstract class GenTaskTracker { + public abstract boolean isValid(); + public abstract Consumer getConsumer(); + } + + final IGenerator generator; + static final class GenTask { + final DhLodPos pos; + final byte dataDetail; + final GenTaskTracker taskTracker; + final CompletableFuture future; + GenTask(DhLodPos pos, byte dataDetail, GenTaskTracker taskTracker, CompletableFuture future) { + this.dataDetail = dataDetail; + this.pos = pos; + this.taskTracker = taskTracker; + this.future = future; + } + } + static final class TaskGroup { + final DhLodPos pos; + byte dataDetail; + final LinkedList members = new LinkedList<>(); // Accessed by gen poller thread only + TaskGroup(DhLodPos pos, byte dataDetail) { + this.pos = pos; + this.dataDetail = dataDetail; + } + + void accept(ChunkSizedData data) { + Iterator iter = members.iterator(); + while (iter.hasNext()) { + GenTask task = iter.next(); + Consumer consumer = task.taskTracker.getConsumer(); + if (consumer == null) { + iter.remove(); + task.future.complete(false); + } else { + consumer.accept(data); + } + } + } + } + static final class InProgressTask { + final TaskGroup group; + CompletableFuture genFuture = null; + InProgressTask(TaskGroup group) { + this.group = group; + } + } + + static class SplitTask extends GenTaskTracker { + final GenTaskTracker parentTracker; + final CompletableFuture parentFuture; + boolean cachedValid = true; + SplitTask(GenTaskTracker parentTracker, CompletableFuture parentFuture) { + this.parentTracker = parentTracker; + this.parentFuture = parentFuture; + } + boolean recheckState() { + if (!cachedValid) return false; + cachedValid = parentTracker.isValid(); + if (!cachedValid) parentFuture.complete(false); + return cachedValid; + } + @Override + public boolean isValid() { + return cachedValid; + } + @Override + public Consumer getConsumer() { + return parentTracker.getConsumer(); + } + } + + private final ConcurrentLinkedQueue looseTasks = new ConcurrentLinkedQueue<>(); + private final HashMap taskGroups = new HashMap<>(); // Accessed by poller only + private final ConcurrentHashMap inProgress = new ConcurrentHashMap<>(); + + private final byte maxGranularity; + private final byte minGranularity; + private final byte maxDataDetail; + private final byte minDataDetail; + private volatile CompletableFuture closer = null; + + public GenerationQueue(IGenerator generator) { + this.generator = generator; + maxGranularity = generator.getMaxGenerationGranularity(); + minGranularity = generator.getMinGenerationGranularity(); + maxDataDetail = generator.getMaxDataDetail(); + minDataDetail = generator.getMinDataDetail(); + if (minGranularity < 4) throw new IllegalArgumentException("DH-IGenerator: min granularity must be at least 4!"); + if (maxGranularity < minGranularity) throw new IllegalArgumentException("DH-IGenerator: max granularity smaller than min granularity!"); + } + + public CompletableFuture submitGenTask(DhLodPos pos, byte requiredDataDetail, GenTaskTracker tracker) { + if (closer != null) return CompletableFuture.completedFuture(false); + if (requiredDataDetail < minDataDetail) { + throw new UnsupportedOperationException("Current generator does not meet requiredDataDetail level"); + } + if (requiredDataDetail > maxDataDetail) requiredDataDetail = maxDataDetail; + + LodUtil.assertTrue(pos.detail > requiredDataDetail+4); + byte granularity = (byte) (pos.detail - requiredDataDetail); + + if (granularity > maxGranularity) { + // Too big of a chunk. We need to split it up + byte subDetail = (byte) (maxGranularity + requiredDataDetail); + int subPosCount = pos.getWidth(subDetail); + DhLodPos cornerSubPos = pos.getCorner(subDetail); + CompletableFuture[] subFutures = new CompletableFuture[subPosCount*subPosCount]; + ArrayList subTasks = new ArrayList<>(subPosCount*subPosCount); + SplitTask splitTask = new SplitTask(tracker, new CompletableFuture<>()); + { + int i = 0; + for (int ox = 0; ox < subPosCount; ox++) { + for (int oz = 0; oz < subPosCount; oz++) { + CompletableFuture subFuture = new CompletableFuture<>(); + subFutures[i++] = subFuture; + subTasks.add(new GenTask(cornerSubPos.offset(ox, oz), requiredDataDetail, splitTask, subFuture)); + } + } + } + CompletableFuture.allOf(subFutures).whenComplete((v,ex) -> { + if (ex != null) splitTask.parentFuture.completeExceptionally(ex); + if (!splitTask.recheckState()) return; // Auto join future + for (CompletableFuture subFuture: subFutures) { + boolean successful = subFuture.join(); + if (!successful) { + splitTask.parentFuture.complete(false); + return; + } + } + splitTask.parentFuture.complete(true); + }); + looseTasks.addAll(subTasks); + if (closer != null) return CompletableFuture.completedFuture(false); + else return splitTask.parentFuture; + } else if (granularity < minGranularity) { + // Too small of a chunk. We'll just over-size the generation. + byte parentDetail = (byte) (minGranularity + requiredDataDetail); + DhLodPos parentPos = pos.convertUpwardsTo(parentDetail); + CompletableFuture future = new CompletableFuture<>(); + looseTasks.add(new GenTask(parentPos, requiredDataDetail, tracker, future)); + if (closer != null) return CompletableFuture.completedFuture(false); + else return future; + } else { + CompletableFuture future = new CompletableFuture<>(); + looseTasks.add(new GenTask(pos, requiredDataDetail, tracker, future)); + if (closer != null) return CompletableFuture.completedFuture(false); + else return future; + } + } + + private void addAndCombineGroup(TaskGroup target) { + byte granularity = (byte) (target.pos.detail - target.dataDetail); + LodUtil.assertTrue(granularity <= maxGranularity && granularity >= minGranularity); + LodUtil.assertTrue(!taskGroups.containsKey(target.pos)); + + // Check and merge all those who has exactly the same dataDetail, and overlaps the position, but have lower granularity than us + if (granularity > minGranularity) { + // TODO: Optimize this check + Iterator groupIter = taskGroups.values().iterator(); + while (groupIter.hasNext()) { + TaskGroup group = groupIter.next(); + if (group.dataDetail != target.dataDetail) continue; + if (!group.pos.overlaps(target.pos)) continue; + + // We should have already ALWAYS selected the higher granularity. + LodUtil.assertTrue(group.pos.detail < target.pos.detail); + groupIter.remove(); // Remove and consume all from that lower granularity request + target.members.addAll(group.members); + } + } + + // Now, Check if we are the missing piece in the 4 quadrants, and if so, combine the four into a new higher granularity group + if (granularity < maxGranularity) { // Obviously, only do so if we aren't at the maxGranularity already + // Check for merging and upping the granularity + DhLodPos corePos = target.pos; + DhLodPos parentPos = corePos.convertUpwardsTo((byte) (corePos.detail+1)); + int targetChildId = target.pos.getChildIndexOfParent(); + boolean allPassed = true; + for (int i = 0; i < 4; i++) { + if (i == targetChildId) continue; + TaskGroup group = taskGroups.get(parentPos.getChild(i)); + if (group == null || group.dataDetail != target.dataDetail) { + allPassed = false; + break; + } + } + if (allPassed) { + LodUtil.assertTrue(!taskGroups.containsKey(parentPos) || taskGroups.get(parentPos).dataDetail != target.dataDetail); + TaskGroup[] groups = new TaskGroup[4]; + for (int i = 0; i < 4; i++) { + if (i==targetChildId) groups[i] = target; + else groups[i] = taskGroups.remove(parentPos.getChild(i)); + LodUtil.assertTrue(groups[i] != null && groups[i].dataDetail == target.dataDetail); + } + + TaskGroup newGroup = taskGroups.get(parentPos); + if (newGroup != null) { + LodUtil.assertTrue(newGroup.dataDetail != target.dataDetail); // if it is equal, we should have been merged ages ago + if (newGroup.dataDetail < target.dataDetail) { + // We can just append us into the existing list. + for (TaskGroup g : groups) newGroup.members.addAll(g.members); + } else { + // We need to upgrade the requested dataDetail of the group. + newGroup.dataDetail = target.dataDetail; + boolean worked = taskGroups.remove(parentPos, newGroup); // Pop it off for later proper merge check + LodUtil.assertTrue(worked); + for (TaskGroup g : groups) newGroup.members.addAll(g.members); + addAndCombineGroup(newGroup); // Recursive check the new group + } + } else { + // There should not be any higher granularity to check, as otherwise we would have merged ages ago + newGroup = new TaskGroup(parentPos, target.dataDetail); + for (TaskGroup g : groups) newGroup.members.addAll(g.members); + addAndCombineGroup(newGroup); // Recursive check the new group + } + return; // We have merged. So no need to add the target group + } + } + + // Finally, we should be safe to add the target group into the list + TaskGroup v = taskGroups.put(target.pos, target); + LodUtil.assertTrue(v == null); // should never be replacing other things + } + + private void processLooseTasks() { + while (!looseTasks.isEmpty()) { + GenTask task = looseTasks.poll(); + byte taskDataDetail = task.dataDetail; + byte taskGranularity = (byte) (task.pos.detail - taskDataDetail); + LodUtil.assertTrue(taskGranularity >= 4 && taskGranularity >= minGranularity && taskGranularity <= maxGranularity); + + // Check existing one + TaskGroup group = taskGroups.get(task.pos); + if (group != null) { + if (group.dataDetail <= taskDataDetail) { + // We can just append us into the existing list. + group.members.add(task); + } else { + // We need to upgrade the requested dataDetail of the group. + group.dataDetail = taskDataDetail; + boolean worked = taskGroups.remove(task.pos, group); // Pop it off for later proper merge check + LodUtil.assertTrue(worked); + group.members.add(task); + addAndCombineGroup(group); + } + } else { + + // Check higher granularity one + byte granularity = taskGranularity; + boolean didAnything = false; + while (++granularity <= maxGranularity) { + group = taskGroups.get(task.pos.convertUpwardsTo((byte) (taskDataDetail + granularity))); + if (group != null && group.dataDetail == taskDataDetail) { + // We can just append to the higher granularity group one + group.members.add(task); + didAnything = true; + break; + } + } + if (!didAnything) { + group = new TaskGroup(task.pos, taskDataDetail); + group.members.add(task); + addAndCombineGroup(group); + } + } + + } + + + } + + private void removeOutdatedGroups() { + // Remove all invalid genTasks and groups + Iterator groupIter = taskGroups.values().iterator(); + while (groupIter.hasNext()) { + TaskGroup group = groupIter.next(); + Iterator taskIter = group.members.iterator(); + while (taskIter.hasNext()) { + GenTask task = taskIter.next(); + if (!task.taskTracker.isValid()) { + taskIter.remove(); + task.future.complete(false); + } + } + if (group.members.isEmpty()) groupIter.remove(); + } + } + + private void pollAndStartNext(DhBlockPos2D targetPos) { + // Select the one with the highest data detail level and closest to the target pos + TaskGroup best = null; + long cachedDist = Long.MAX_VALUE; + for (TaskGroup group : taskGroups.values()) { + if (best != null) { + if (group.dataDetail < best.dataDetail) continue; + long dist = group.pos.getCenter().distSquared(targetPos); + if (cachedDist <= dist) continue; + cachedDist = dist; + } + best = group; + } + if (best != null) { + InProgressTask startedTask = new InProgressTask(best); + InProgressTask casTask = inProgress.putIfAbsent(best.pos, startedTask); + boolean worked = taskGroups.remove(best.pos, best); // Remove the selected task from the group + LodUtil.assertTrue(worked); + if (casTask != null) { + // Note: Due to concurrency reasons, even if the currently running task is compatible with selected task, + // we cannot use it, as some chunks may have already been written into. + pollAndStartNext(targetPos); // Poll next one. + TaskGroup exchange = taskGroups.put(best.pos, best); // put back the task. + LodUtil.assertTrue(exchange == null); + } else { + startTaskGroup(startedTask); + } + } + + } - public GenerationQueue() {} public void pollAndStartClosest(DhBlockPos2D targetPos) { if (generator == null) throw new IllegalStateException("generator is null"); if (generator.isBusy()) return; - DhLodPos closest = null; - long closestDist = Long.MAX_VALUE; - int smallestDetail = Integer.MAX_VALUE; - for (DhLodPos key : taskMap.keySet()) { - if (key.detail > smallestDetail) continue; - long dist = key.getCenter().distSquared(targetPos); - if (key.detail == smallestDetail && dist >= closestDist) continue; - closest = key; - closestDist = dist; - smallestDetail = key.detail; - } - if (closest != null) { - CompletableFuture future = taskMap.remove(closest); - startFuture(closest, future); - } + removeOutdatedGroups(); + processLooseTasks(); + pollAndStartNext(targetPos); } - public void setGenerator(IGenerator generator) { - LodUtil.assertTrue(generator != null); - LodUtil.assertTrue(this.generator == null); - this.generator = generator; - inProgress.set(new ConcurrentHashMap<>(16)); - } - public void removeGenerator() { - LodUtil.assertTrue(generator != null); - this.generator = null; - ConcurrentHashMap> swapped = this.inProgress.getAndSet(null); - swapped.forEach((k,f) -> f.cancel(true)); - } + private void startTaskGroup(InProgressTask task) { + byte dataDetail = task.group.dataDetail; + DhLodPos pos = task.group.pos; + byte granularity = (byte) (pos.detail - dataDetail); + LodUtil.assertTrue(granularity >= minGranularity && granularity <= maxGranularity); + LodUtil.assertTrue(dataDetail >= minDataDetail && dataDetail <= maxDataDetail); - private CompletableFuture createFuture(DhLodPos pos) { - logger.info("Creating gen future for {}", pos); - CompletableFuture future = new CompletableFuture<>(); - CompletableFuture swapped = taskMap.put(pos, future); - LodUtil.assertTrue(swapped == null); - return future; - } - - private void startFuture(DhLodPos pos, CompletableFuture resultFuture) { - byte dataDetail = generator.getDataDetail(); - byte minGenGranularity = generator.getMinGenerationGranularity(); - byte maxGenGranularity = generator.getMaxGenerationGranularity(); - if (minGenGranularity < 4 || maxGenGranularity < 4) { - throw new IllegalStateException("Generation granularity must be at least 4!"); - } - byte minUnitDetail = (byte) (dataDetail + minGenGranularity); - byte maxUnitDetail = (byte) (dataDetail + maxGenGranularity); - LodUtil.assertTrue(pos.detail >= minUnitDetail && pos.detail <= maxUnitDetail); - byte genGranularity = (byte) (pos.detail - dataDetail); DHChunkPos chunkPosMin = new DHChunkPos(pos.getCorner()); - logger.info("Generating section {} with granularity {} at {}", pos, genGranularity, chunkPosMin); - int perCallChunksWidth = 1 << (genGranularity - 4); - - CompletableFuture> dataFuture = generator.generate(chunkPosMin, genGranularity); - final ConcurrentHashMap> map = this.inProgress.get(); - map.put(pos, //FIXME: Slight race condition issue here with map.clear()! - dataFuture.handle((data, ex) -> { - if (ex != null) { - if (ex instanceof CompletionException) { - ex = ex.getCause(); - } - UncheckedInterruptedException.rethrowIfIsInterruption(ex); - logger.error("Error generating data for section {}", pos, ex); - throw new CompletionException("Generation failed", ex); - } - LodUtil.assertTrue(data != null); - if (data.gridSize < (1 << (genGranularity-4))) { - logger.error( - "Generator at {} returned {} by {} chunks but requested granularity was {}, which expect at least {} by {} chunks! ", - pos, data.gridSize, data.gridSize, genGranularity, perCallChunksWidth, perCallChunksWidth); - throw new RuntimeException("Generation failed. Generator returned less data than requested!"); - } - logger.info("Completed generating {} by {} chunks to sections that overlaps {}", - data.gridSize, data.gridSize, pos); - return data; - }).thenApply((list) -> { - GenerationResult result = new GenerationResult(); - result.dataList.addAll(list); - return result; - }).handle((r, e) -> { - if (e!=null) resultFuture.completeExceptionally(e); else resultFuture.complete(r); - map.remove(pos); - return null; - }) - ); + logger.info("Generating section {} with granularity {} at {}", pos, granularity, chunkPosMin); + task.genFuture = generator.generate( + chunkPosMin, granularity, dataDetail, task.group::accept); + task.genFuture.whenComplete((v, ex) -> { + if (ex != null) { + logger.error("Error generating data for section {}", pos, ex); + task.group.members.forEach(m -> m.future.complete(false)); + } else { + logger.info("Section generation at {} complated", pos); + task.group.members.forEach(m -> m.future.complete(true)); + } + boolean worked = inProgress.remove(pos, task); + LodUtil.assertTrue(worked); + }); } - public CompletableFuture generate(DhSectionPos sectionPos) { - byte maxGen = (byte) (generator.getMaxGenerationGranularity() + generator.getDataDetail()); - if (sectionPos.sectionDetail > maxGen) { - int count = 1 << (sectionPos.sectionDetail - maxGen); - DhLodPos minPos = sectionPos.getCorner(maxGen); - ArrayList> futures = new ArrayList<>(count*count); - for (int x = 0; x < count; x++) { - for (int z = 0; z < count; z++) { - DhLodPos subPos = new DhLodPos(maxGen, minPos.x + x, minPos.z + z); - futures.add(cqcpTree.createOrUseExisting(subPos, this::createFuture)); - } - } - // FIXME: Does `allOf` have correct behaviour when one of the futures fails? - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .thenApply((v) -> { - FullDataSource newSource = FullDataSource.createEmpty(sectionPos); - for (CompletableFuture future : futures) { - try { - GenerationResult result = future.join(); - for (ChunkSizedData data : result.dataList) { - if (data.getBBoxLodPos().overlaps(sectionPos.getSectionBBoxPos())) newSource.update(data); - } - } catch (Exception e) { - UncheckedInterruptedException.rethrowIfIsInterruption(e); - // else log - logger.error("Error generating data for section {}", sectionPos, e); - } - } - return newSource; - }); - } else { - DhLodPos lodPos = sectionPos.getSectionBBoxPos(); - return cqcpTree.createOrUseExisting(lodPos, this::createFuture).thenApply( - (result) -> { - if (result == null || result.dataList.isEmpty()) return FullDataSource.createEmpty(sectionPos); - FullDataSource newSource = FullDataSource.createEmpty(sectionPos); - for (ChunkSizedData data : result.dataList) { - if (data.getBBoxLodPos().overlaps(sectionPos.getSectionBBoxPos())) newSource.update(data); - } - return newSource; - }); + public CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning) { + taskGroups.values().forEach(g -> g.members.forEach(t -> t.future.complete(false))); + taskGroups.clear(); + ArrayList> array = new ArrayList<>(inProgress.size()); + inProgress.values().forEach(runningTask -> array.add(runningTask.genFuture)); + closer = CompletableFuture.allOf(array.toArray(CompletableFuture[]::new)); + if (cancelCurrentGeneration) { + array.forEach(f -> f.cancel(alsoInterruptRunning)); } + looseTasks.forEach(t -> t.future.complete(false)); + looseTasks.clear(); + return closer; } @Override public void close() { - //TODO + if (closer == null) startClosing(true, true); + LodUtil.assertTrue(closer != null); + try { + closer.orTimeout(SHUTDOWN_TIMEOUT_SEC, TimeUnit.SECONDS).join(); + } catch (Throwable e) { + logger.error("Failed to close generation queue: ", e); + } } - -// -// DhBlockPos2D lastPlayerPos = new DhBlockPos2D(0, 0); -// final ConcurrentHashMap> trackers = new ConcurrentHashMap<>(); -// final BiConsumer writeConsumer; -// final HashSet> inProgressSections = new HashSet<>(); - -// -// public void track(PlaceHolderRenderSource source) { -// //logger.info("Tracking source {} at {}", source, source.getSectionPos()); -// trackers.put(source.getSectionPos(), new WeakReference<>(source)); -// } -// -// private void update() { -// LinkedList toRemove = new LinkedList<>(); -// for (DhSectionPos pos : trackers.keySet()) { -// WeakReference ref = trackers.get(pos); -// if (ref.get() == null) { -// toRemove.add(pos); -// } -// } -// for (DhSectionPos pos : toRemove) { -// trackers.remove(pos); -// } -// } - -// //FIXME: Do optimizations on polling closest to player. (Currently its a O(n) search!) -// //FIXME: Do not return sections that is already being generated. -// //FIXME: Optimize the checks for inProgressSections. -// private DhSectionPos pollClosest(DhBlockPos2D playerPos) { -// update(); -// DhSectionPos closest = null; -// long closestDist = Long.MAX_VALUE; -// for (DhSectionPos pos : trackers.keySet()) { -// if (inProgressSections.contains(pos)) { -// continue; -// } -// long distSqr = pos.getCenter().getCenter().distSquared(playerPos); -// if (distSqr < closestDist) { -// closest = pos; -// closestDist = distSqr; -// } -// } -// if (closest != null) inProgressSections.add(closest); -// return closest; -// } } diff --git a/core/src/main/java/com/seibel/lod/core/a7/generation/IChunkGenerator.java b/core/src/main/java/com/seibel/lod/core/a7/generation/IChunkGenerator.java index aac2ce518..0969ccc83 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/generation/IChunkGenerator.java +++ b/core/src/main/java/com/seibel/lod/core/a7/generation/IChunkGenerator.java @@ -7,16 +7,15 @@ import com.seibel.lod.core.util.gridList.ArrayGridList; import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; public interface IChunkGenerator extends IGenerator { - CompletableFuture> generateChunks(DHChunkPos chunkPosMin, byte granularity); + CompletableFuture generateChunks(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer); @Override - default CompletableFuture> generate(DHChunkPos chunkPosMin, byte granularity) { - return generateChunks(chunkPosMin, granularity).thenApply(chunks -> { - ArrayGridList chunkData = new ArrayGridList<>(chunks.gridSize); - chunks.forEachPos((x, y) -> chunkData.set(x, y, LodDataBuilder.createChunkData(chunks.get(x, y)))); - return chunkData; + default CompletableFuture generate(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer) { + return generateChunks(chunkPosMin, granularity, targetDataDetail, (chunk) -> { + resultConsumer.accept(LodDataBuilder.createChunkData(chunk)); }); } diff --git a/core/src/main/java/com/seibel/lod/core/a7/generation/IGenerator.java b/core/src/main/java/com/seibel/lod/core/a7/generation/IGenerator.java index 11a4c3e60..70e3583d2 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/generation/IGenerator.java +++ b/core/src/main/java/com/seibel/lod/core/a7/generation/IGenerator.java @@ -5,18 +5,22 @@ import com.seibel.lod.core.objects.DHChunkPos; import com.seibel.lod.core.util.gridList.ArrayGridList; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; public interface IGenerator extends AutoCloseable { // What is the detail / resolution of the data? (This will offset the generation granularity) // (minimum detail is 0, maximum detail is 255) (though that high isn't really... realistic) // (0 = 1x1 block per data, 1 = 2x2 block per data, 2 = 4x4 block per data... etc.) // TODO: System currently only supports 1x1 block per data. - byte getDataDetail(); + byte getMinDataDetail(); + byte getMaxDataDetail(); + int getPriority(); // What is the min batch size of a single generation? // (minimum return value is 4 since that's the MC chunk size) // (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.) byte getMinGenerationGranularity(); - // What is the max batch size of a single generation? + + // What is the max batch size of a single generation? The system will try to group tasks to the max batch size if possible // (minimum return value is 4 since that's the MC chunk size) // (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.) byte getMaxGenerationGranularity(); @@ -24,7 +28,7 @@ public interface IGenerator extends AutoCloseable { // Start a generation event // (Note that the chunkPos is always aligned to the granularity) // (For example, if the granularity is 4, data detail is 0, the chunkPos will be aligned to 16x16 blocks) - CompletableFuture> generate(DHChunkPos chunkPosMin, byte granularity); + CompletableFuture generate(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer); // Return whether the generator is currently busy and cannot accept new generation requests. boolean isBusy(); diff --git a/core/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java index d0fe7eb5f..cadfde12b 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java @@ -1,7 +1,6 @@ package com.seibel.lod.core.a7.level; import com.seibel.lod.core.a7.generation.GenerationQueue; -import com.seibel.lod.core.a7.generation.IGenerator; import com.seibel.lod.core.a7.render.LodQuadTree; import com.seibel.lod.core.a7.save.io.file.GeneratedDataFileHandler; import com.seibel.lod.core.a7.util.FileScanner; @@ -24,6 +23,7 @@ import com.seibel.lod.core.wrapperInterfaces.world.IBiomeWrapper; import com.seibel.lod.core.wrapperInterfaces.world.IClientLevelWrapper; import com.seibel.lod.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.lod.core.wrapperInterfaces.world.IServerLevelWrapper; +import net.minecraft.world.entity.ambient.Bat; import org.apache.logging.log4j.Logger; import java.util.concurrent.CompletableFuture; @@ -32,23 +32,22 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); public final LocalSaveStructure save; - public final DataFileHandler dataFileHandler; - public GenerationQueue generationQueue = null; + public final GeneratedDataFileHandler dataFileHandler; + public volatile GenerationQueue generationQueue = null; public RenderFileHandler renderFileHandler = null; public RenderBufferHandler renderBufferHandler = null; //TODO: Should this be owned by renderer? public final IServerLevelWrapper serverLevel; public IClientLevelWrapper clientLevel; public a7LodRenderer renderer = null; public LodQuadTree tree = null; - public BatchGenerator worldGenerator = null; + public volatile BatchGenerator worldGenerator = null; public DhClientServerLevel(LocalSaveStructure save, IServerLevelWrapper level) { this.serverLevel = level; this.save = save; save.getDataFolder(level).mkdirs(); save.getRenderCacheFolder(level).mkdirs(); - generationQueue = new GenerationQueue(); - dataFileHandler = new GeneratedDataFileHandler(this, save.getDataFolder(level), generationQueue); + dataFileHandler = new GeneratedDataFileHandler(this, save.getDataFolder(level)); FileScanner.scanFile(save, serverLevel, dataFileHandler, null); LOGGER.info("Started DHLevel for {} with saves at {}", level, save); } @@ -74,7 +73,8 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { this.clientLevel = clientLevel; // TODO: Make a registry for generators for modding support. worldGenerator = new BatchGenerator(this); - generationQueue.setGenerator(worldGenerator); + generationQueue = new GenerationQueue(worldGenerator); + dataFileHandler.setGenerationQueue(generationQueue); renderFileHandler = new RenderFileHandler(dataFileHandler, this, save.getRenderCacheFolder(serverLevel)); tree = new LodQuadTree(this, Config.Client.Graphics.Quality.lodChunkRenderDistance.get()*16, MC_CLIENT.getPlayerBlockPos().x, MC_CLIENT.getPlayerBlockPos().z, renderFileHandler); @@ -102,18 +102,25 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { } tree.close(); tree = null; - generationQueue.removeGenerator(); - try { - worldGenerator.close(); - } catch (Exception e) { - LOGGER.error("Error closing world generator", e); - } + dataFileHandler.popGenerationQueue(); + final BatchGenerator f_worldGen = worldGenerator; + CompletableFuture closer = generationQueue.startClosing(true, true) + .exceptionally(ex -> { + LOGGER.error("Error closing geberation queue", ex); + return null; + }).thenRun(f_worldGen::close) + .exceptionally(ex -> { + LOGGER.error("Error closing geberation queue", ex); + return null; + }); + generationQueue = null; worldGenerator = null; renderBufferHandler.close(); renderBufferHandler = null; renderFileHandler.flushAndSave(); //Ignore the completion feature so that this action is async renderFileHandler.close(); renderFileHandler = null; + closer.join(); // TODO: Could this cause deadlocks? we are blocking in main thread. } @Override @@ -158,6 +165,7 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { } @Override public void close() { + if (generationQueue != null) generationQueue.close(); if (worldGenerator != null) worldGenerator.close(); if (renderer != null) renderer.close(); if (tree != null) tree.close(); @@ -170,10 +178,12 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { @Override public void doWorldGen() { - if (worldGenerator != null) { - worldGenerator.update(); - if (generationQueue != null) - generationQueue.pollAndStartClosest(new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos())); + final BatchGenerator f_worldGen = worldGenerator; + if (f_worldGen != null) { + f_worldGen.update(); + final GenerationQueue f_genQueue = generationQueue; + if (f_genQueue != null) + f_genQueue.pollAndStartClosest(new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos())); } } diff --git a/core/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java b/core/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java index f643eaab0..92f40b2eb 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java @@ -1,5 +1,6 @@ package com.seibel.lod.core.a7.level; +import com.seibel.lod.core.a7.save.io.file.RemoteDataFileHandler; import com.seibel.lod.core.a7.util.FileScanner; import com.seibel.lod.core.a7.save.io.file.DataFileHandler; import com.seibel.lod.core.a7.save.structure.LocalSaveStructure; @@ -22,7 +23,7 @@ public class DhServerLevel implements IServerLevel this.save = save; this.level = level; save.getDataFolder(level).mkdirs(); - dataFileHandler = new DataFileHandler(this, save.getDataFolder(level), null); //FIXME: GenerationQueue + dataFileHandler = new DataFileHandler(this, save.getDataFolder(level)); //FIXME: GenerationQueue FileScanner.scanFile(save, level, dataFileHandler, null); LOGGER.info("Started DHLevel for {} with saves at {}", level, save); } diff --git a/core/src/main/java/com/seibel/lod/core/a7/pos/DhLodPos.java b/core/src/main/java/com/seibel/lod/core/a7/pos/DhLodPos.java index b2a04753c..88c7be0b5 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/pos/DhLodPos.java +++ b/core/src/main/java/com/seibel/lod/core/a7/pos/DhLodPos.java @@ -94,6 +94,9 @@ public class DhLodPos implements Comparable { if (width.detail < detail) throw new IllegalArgumentException("add called with width.detail < pos detail"); return new DhLodPos(detail, x + width.convertTo(detail).value, z + width.convertTo(detail).value); } + public DhLodPos offset(int ox, int oz) { + return new DhLodPos(detail, x+ox, z+oz); + } @Override public int compareTo(@NotNull DhLodPos o) { diff --git a/core/src/main/java/com/seibel/lod/core/a7/render/LodRenderSection.java b/core/src/main/java/com/seibel/lod/core/a7/render/LodRenderSection.java index 5d684c9a5..11f2ebbe4 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/lod/core/a7/render/LodRenderSection.java @@ -69,6 +69,7 @@ public class LodRenderSection { } } if (lodRenderSource != null) { + provider.refreshRenderSource(lodRenderSource); lodRenderSource.flushWrites(level); } } diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java index d175bb2bb..d7b817d81 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java @@ -2,14 +2,13 @@ package com.seibel.lod.core.a7.save.io; import java.io.*; import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Consumer; +import java.util.concurrent.atomic.AtomicLong; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; @@ -36,7 +35,7 @@ public class MetaFile { // // 8 bytes: datatype identifier // - // 8 bytes: timestamp + // 8 bytes: data version // Used size: 40 bytes // Remaining space: 24 bytes @@ -52,28 +51,38 @@ public class MetaFile { public final DhSectionPos pos; public File path; - public int checksum; - public long timestamp; - public byte dataLevel; - //Loader stuff - public long dataTypeId; - public byte loaderVersion; + @FunctionalInterface + public interface IOConsumer { + void accept(T t) throws IOException; + } - private final ReentrantReadWriteLock assertLock = new ReentrantReadWriteLock(); + public static class MetaData { + public DhSectionPos pos; + public int checksum; + public AtomicLong dataVersion; + public byte dataLevel; + //Loader stuff + public long dataTypeId; + public byte loaderVersion; - // Load a metaFile in this path. It also automatically read the metadata. - protected MetaFile(File path) throws IOException { - this.path = path; - validateFile(); - LodUtil.assertTrue(assertLock.readLock().tryLock()); - try (FileChannel channel = FileChannel.open(path.toPath(), StandardOpenOption.READ)) { + public MetaData(DhSectionPos pos, int checksum, long dataVersion, byte dataLevel, long dataTypeId, byte loaderVersion) { + this.pos = pos; + this.checksum = checksum; + this.dataVersion = new AtomicLong(dataVersion); + this.dataLevel = dataLevel; + this.dataTypeId = dataTypeId; + this.loaderVersion = loaderVersion; + } + } + public volatile MetaData metaData = null; + private static MetaData readMeta(File file) throws IOException { + try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE); channel.read(buffer, 0); channel.close(); buffer.flip(); - this.path = path; int magic = buffer.getInt(); if (magic != METADATA_MAGIC_BYTES) { throw new IOException("Invalid file: Magic bytes check failed."); @@ -81,26 +90,19 @@ public class MetaFile { int x = buffer.getInt(); int y = buffer.getInt(); // Unused int z = buffer.getInt(); - checksum = buffer.getInt(); + int checksum = buffer.getInt(); byte detailLevel = buffer.get(); - dataLevel = buffer.get(); - loaderVersion = buffer.get(); + byte dataLevel = buffer.get(); + byte loaderVersion = buffer.get(); byte unused = buffer.get(); - dataTypeId = buffer.getLong(); - timestamp = buffer.getLong(); + long dataTypeId = buffer.getLong(); + long timestamp = buffer.getLong(); LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE); - pos = new DhSectionPos(detailLevel, x, z); - } finally { - assertLock.readLock().unlock(); + DhSectionPos dataPos = new DhSectionPos(detailLevel, x, z); + return new MetaData(dataPos, checksum, timestamp, dataLevel, dataTypeId, loaderVersion); } } - // Make a new MetaFile. It doesn't load or write any metadata itself. - protected MetaFile(File path, DhSectionPos pos) { - this.path = path; - this.pos = pos; - } - private void validateFile() throws IOException { if (!path.exists()) throw new IOException("File missing"); if (!path.isFile()) throw new IOException("Not a file"); @@ -108,42 +110,32 @@ public class MetaFile { if (!path.canWrite()) throw new IOException("File not writable"); } - protected void updateMetaData() throws IOException { + // Create a metaFile in this path. If the path has a file, throws FileAlreadyExistsException + protected MetaFile(File path, DhSectionPos pos) throws IOException { + this.path = path; + this.pos = pos; + if (path.exists()) throw new FileAlreadyExistsException(path.toString()); + } + // Load a metaFile in this path + protected MetaFile(File path) throws IOException { + this.path = path; + if (!path.exists()) throw new FileNotFoundException("File not found at " + path); validateFile(); - LodUtil.assertTrue(assertLock.readLock().tryLock()); - try (FileChannel channel = FileChannel.open(path.toPath(), StandardOpenOption.READ)) { - ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE); - channel.read(buffer, 0); - channel.close(); - buffer.flip(); + metaData = readMeta(path); + pos = metaData.pos; + } - int magic = buffer.getInt(); - if (magic != METADATA_MAGIC_BYTES) { - throw new IOException("Invalid file: Magic bytes check failed."); - } - int x = buffer.getInt(); - int y = buffer.getInt(); // Unused - int z = buffer.getInt(); - checksum = buffer.getInt(); - byte detailLevel = buffer.get(); - dataLevel = buffer.get(); - byte loaderVersion = buffer.get(); - byte unused = buffer.get(); - dataTypeId = buffer.getLong(); - timestamp = buffer.getLong(); - LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE); - - DhSectionPos newPos = new DhSectionPos(detailLevel, x, z); - if (!newPos.equals(pos)) { - throw new IOException("Invalid file: Section position changed."); - } - this.loaderVersion = loaderVersion; - } finally { - assertLock.readLock().unlock(); + protected void loadMetaData() throws IOException { + validateFile(); + metaData = readMeta(path); + if (!metaData.pos.equals(pos)) { + LOGGER.warn("The file is from a different location than expected! Expects {} but got {}. Ignoring file tag.", pos, metaData.pos); + metaData.pos = pos; } } - protected void writeData(Consumer dataWriter) throws IOException { + protected void writeData(IOConsumer dataWriter) throws IOException { + LodUtil.assertTrue(metaData != null); if (path.exists()) validateFile(); File writerFile; if (USE_ATOMIC_MOVE_REPLACE) { @@ -152,7 +144,7 @@ public class MetaFile { } else { writerFile = path; } - LodUtil.assertTrue(assertLock.writeLock().tryLock()); + try (FileChannel file = FileChannel.open(writerFile.toPath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { { @@ -173,11 +165,11 @@ public class MetaFile { buff.putInt(pos.sectionZ); buff.putInt(checksum); buff.put(pos.sectionDetail); - buff.put(dataLevel); - buff.put(loaderVersion); + buff.put(metaData.dataLevel); + buff.put(metaData.loaderVersion); buff.put(Byte.MIN_VALUE); // Unused - buff.putLong(dataTypeId); - buff.putLong(timestamp); + buff.putLong(metaData.dataTypeId); + buff.putLong(metaData.dataVersion.get()); LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE); buff.flip(); file.write(buff); @@ -188,7 +180,6 @@ public class MetaFile { Files.move(writerFile.toPath(), path.toPath(), StandardCopyOption.ATOMIC_MOVE); } } finally { - assertLock.writeLock().unlock(); try { if (USE_ATOMIC_MOVE_REPLACE && writerFile.exists()) { boolean i = writerFile.delete(); // Delete temp file. Ignore errors if fails. diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java index 0d90e1427..025ef7cef 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java @@ -4,10 +4,11 @@ import com.google.common.collect.HashMultimap; import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullDataSource; +import com.seibel.lod.core.a7.datatype.full.SparseDataSource; import com.seibel.lod.core.a7.level.ILevel; -import com.seibel.lod.core.a7.level.IServerLevel; import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.a7.save.io.MetaFile; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; @@ -20,9 +21,10 @@ import java.util.Collections; import java.util.Comparator; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; +import java.util.function.Consumer; public class DataFileHandler implements IDataSourceProvider { // Note: Single main thread only for now. May make it multi-thread later, depending on the usage. @@ -33,14 +35,10 @@ public class DataFileHandler implements IDataSourceProvider { final File saveDir; AtomicInteger topDetailLevel = new AtomicInteger(-1); final int minDetailLevel = FullDataSource.SECTION_SIZE_OFFSET; - final Function> dataSourceCreator; - - public DataFileHandler(ILevel level, File saveRootDir, - Function> dataSourceCreator) { + public DataFileHandler(ILevel level, File saveRootDir) { this.saveDir = saveRootDir; this.level = level; - this.dataSourceCreator = dataSourceCreator; } /* @@ -55,7 +53,7 @@ public class DataFileHandler implements IDataSourceProvider { { // Sort files by pos. for (File file : detectedFiles) { try { - DataMetaFile metaFile = new DataMetaFile(level, file); + DataMetaFile metaFile = new DataMetaFile(this, level, file); filesByPos.put(metaFile.pos, metaFile); } catch (IOException e) { LOGGER.error("Failed to read file {}. File will be deleted.", file, e); @@ -71,7 +69,7 @@ public class DataFileHandler implements IDataSourceProvider { Collection metaFiles = filesByPos.get(pos); DataMetaFile fileToUse; if (metaFiles.size() > 1) { - fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.timestamp)); + fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get())); { StringBuilder sb = new StringBuilder(); sb.append("Multiple files with the same pos: "); @@ -108,26 +106,23 @@ public class DataFileHandler implements IDataSourceProvider { } } - private DataMetaFile atomicGetOrMakeFile(DhSectionPos pos) { + protected DataMetaFile atomicGetOrMakeFile(DhSectionPos pos) { DataMetaFile metaFile = files.get(pos); if (metaFile == null) { - File file = computeDefaultFilePath(pos); - //FIXME: Handle file already exists issue. Possibly by renaming the file. - LodUtil.assertTrue(!file.exists(), "File {} already exist for path {}", file, pos); - CompletableFuture gen = new CompletableFuture<>(); - DataMetaFile newMetaFile = new DataMetaFile(level, file, pos, gen); - metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. - if (metaFile == null) { - buildFile(pos, gen); - metaFile = newMetaFile; - } else { - gen.cancel(true); + DataMetaFile newMetaFile; + try { + newMetaFile = new DataMetaFile(this, level, pos); + } catch (IOException e) { + LOGGER.error("IOException on creating new data file at {}", pos, e); + return null; } + metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. + if (metaFile == null) metaFile = newMetaFile; } return metaFile; } - private void selfSearch(DhSectionPos basePos, DhSectionPos pos, ArrayList existFiles, ArrayList missing) { + protected void selfSearch(DhSectionPos basePos, DhSectionPos pos, ArrayList existFiles, ArrayList missing) { byte detail = pos.sectionDetail; boolean allEmpty = true; outerLoop: @@ -210,48 +205,6 @@ public class DataFileHandler implements IDataSourceProvider { } - private void buildFile(DhSectionPos pos, CompletableFuture gen) { - ArrayList existFiles = new ArrayList<>(); - ArrayList missing = new ArrayList<>(); - selfSearch(pos, pos, existFiles, missing); - LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty()); - if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) { - dataSourceCreator.apply(pos).whenComplete((f, ex) -> { - if (ex != null) { - gen.completeExceptionally(ex); - } else { - gen.complete(f); - } - }); - return; - } - - - LOGGER.info("Creating file at {} using {} existing files and {} new files.", pos, existFiles.size(), missing.size()); - ArrayList> futures = new ArrayList<>(existFiles.size() + missing.size()); - for (DhSectionPos missingPos : missing) { - existFiles.add(atomicGetOrMakeFile(missingPos)); - } - FullDataSource fullDataSource = FullDataSource.createEmpty(pos); - for (DataMetaFile metaFile : existFiles) { - futures.add( - metaFile.loadOrGetCached(fileReaderThread).whenComplete((data, ex) -> { - if (ex != null) return; - if (!(data instanceof FullDataSource)) return; - fullDataSource.writeFromLower((FullDataSource) data); - }) - ); - } - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .whenComplete((v, ex) -> { - if (ex != null) { - gen.completeExceptionally(ex); - } else { - gen.complete(fullDataSource); - } - }); - } - /* * This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ @@ -259,7 +212,8 @@ public class DataFileHandler implements IDataSourceProvider { public CompletableFuture read(DhSectionPos pos) { topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetail)); DataMetaFile metaFile = atomicGetOrMakeFile(pos); - return metaFile.loadOrGetCached(fileReaderThread); + if (metaFile == null) return CompletableFuture.completedFuture(null); + return metaFile.loadOrGetCached(); } /* @@ -289,23 +243,81 @@ public class DataFileHandler implements IDataSourceProvider { public CompletableFuture flushAndSave() { ArrayList> futures = new ArrayList<>(); for (DataMetaFile metaFile : files.values()) { - futures.add(metaFile.flushAndSave(fileReaderThread)); + futures.add(metaFile.flushAndSave()); } return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } @Override - public boolean isCacheValid(DhSectionPos sectionPos, long timestamp) { + public long getLatestCacheVersion(DhSectionPos sectionPos) { DataMetaFile file = files.get(sectionPos); - if (file == null) return false; - //TODO - return true; + if (file == null) return 0; + return file.getDataVersion(); } - private File computeDefaultFilePath(DhSectionPos pos) { //TODO: Temp code as we haven't decided on the file naming & location yet. + @Override + public CompletableFuture onCreateDataFile(DataMetaFile file) { + DhSectionPos pos = file.pos; + ArrayList existFiles = new ArrayList<>(); + ArrayList missing = new ArrayList<>(); + selfSearch(pos, pos, existFiles, missing); + LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty()); + if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) { + // None exist. + SparseDataSource dataSource = SparseDataSource.createEmpty(pos); + return CompletableFuture.completedFuture(dataSource); + } else { + + for (DhSectionPos missingPos : missing) { + DataMetaFile newfile = atomicGetOrMakeFile(missingPos); + if (newfile != null) existFiles.add(newfile); + } + final ArrayList> futures = new ArrayList<>(existFiles.size()); + final SparseDataSource dataSource = SparseDataSource.createEmpty(pos); + + for (DataMetaFile f : existFiles) { + futures.add(f.loadOrGetCached() + .exceptionally((ex) -> null) + .thenAccept((data) -> { + if (data != null) { + if (data instanceof SparseDataSource) + dataSource.sampleFrom((SparseDataSource) data); + else if (data instanceof FullDataSource) + dataSource.sampleFrom((FullDataSource) data); + else LodUtil.assertNotReach(); + } + }) + ); + } + return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)) + .thenApply((v) -> dataSource.trySelfPromote()); + } + } + + @Override + public LodDataSource onDataFileLoaded(LodDataSource source, Consumer updater) { + updater.accept(source); + if (source instanceof SparseDataSource) return ((SparseDataSource) source).trySelfPromote(); + return source; + } + @Override + public LodDataSource onDataFileRefresh(LodDataSource source, Consumer updater) { + updater.accept(source); + if (source instanceof SparseDataSource) return ((SparseDataSource) source).trySelfPromote(); + return source; + } + + @Override + public File computeDataFilePath(DhSectionPos pos) { return new File(saveDir, pos.serialize() + ".lod"); } + @Override + public Executor getIOExecutor() { + return fileReaderThread; + } + + @Override public void close() { DataMetaFile.debugCheck(); diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java index e1ebb1451..f27eca82e 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java @@ -1,28 +1,19 @@ package com.seibel.lod.core.a7.save.io.file; -import java.awt.*; import java.io.*; import java.lang.ref.*; -import java.security.Provider; -import java.sql.Ref; -import java.util.List; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Executor; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; -import java.util.function.Supplier; import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.DataSourceLoader; +import com.seibel.lod.core.a7.datatype.LodRenderSource; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; -import com.seibel.lod.core.a7.datatype.full.FullDataSource; import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.a7.save.io.MetaFile; import com.seibel.lod.core.a7.level.ILevel; @@ -31,14 +22,15 @@ import com.seibel.lod.core.util.LodUtil; public class DataMetaFile extends MetaFile { private final ILevel level; + private final IDataSourceProvider handler; + private boolean doesFileExist; + public DataSourceLoader loader; public Class dataType; - AtomicInteger localVersion = new AtomicInteger(); // This MUST be atomic - // The '?' type should either be: - // SoftReference, or - Non-dirty file that can be GCed - // CompletableFuture, or - File that is being loaded - // null - Nothing is loaded or being loaded + // SoftReference, or - Non-dirty file that can be GCed + // CompletableFuture, or - File that is being loaded. No guarantee that the type is promotable or not + // null - Nothing is loaded or being loaded AtomicReference data = new AtomicReference(null); //TODO: use ConcurrentAppendSingleSwapContainer instead of below: @@ -46,14 +38,18 @@ public class DataMetaFile extends MetaFile { ReentrantReadWriteLock appendLock = new ReentrantReadWriteLock(); ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); } + + // ===Concurrent Write stuff=== AtomicReference writeQueue = new AtomicReference<>(new GuardedMultiAppendQueue()); GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue(); - private final AtomicBoolean inCacheWriteLock = new AtomicBoolean(false); + // =========================== + private final AtomicBoolean inCacheWriteAccessAsserter = new AtomicBoolean(false); + + // ===Object lifetime stuff=== private static final ReferenceQueue lifeCycleDebugQueue = new ReferenceQueue<>(); private static final Set lifeCycleDebugSet = ConcurrentHashMap.newKeySet(); - private static class DataObjTracker extends PhantomReference implements Closeable { private final DhSectionPos pos; DataObjTracker(LodDataSource data) { @@ -67,6 +63,50 @@ public class DataMetaFile extends MetaFile { lifeCycleDebugSet.remove(this); } } + // =========================== + + + + // Create a new metaFile + public DataMetaFile(IDataSourceProvider handler, ILevel level, DhSectionPos pos) throws IOException { + super(handler.computeDataFilePath(pos), pos); + debugCheck(); + this.handler = handler; + this.level = level; + LodUtil.assertTrue(metaData == null); + doesFileExist = false; + } + + public DataMetaFile(IDataSourceProvider handler, ILevel level, File path) throws IOException { + super(path); + debugCheck(); + this.handler = handler; + this.level = level; + LodUtil.assertTrue(metaData != null); + loader = DataSourceLoader.getLoader(metaData.dataTypeId, metaData.loaderVersion); + if (loader == null) { + throw new IOException("Invalid file: Data type loader not found: " + + metaData.dataTypeId + "(v" + metaData.loaderVersion + ")"); + } + dataType = loader.clazz; + doesFileExist = true; + } + + public CompletableFuture flushAndSave() { + debugCheck(); + boolean isEmpty = writeQueue.get().queue.isEmpty(); + if (!isEmpty) { + return loadOrGetCached().thenApply((unused) -> null); // This will flush the data to disk. + } else { + return CompletableFuture.completedFuture(null); + } + } + + public long getDataVersion() { + debugCheck(); + MetaData getData = metaData; + return getData == null ? 0 : metaData.dataVersion.get(); + } public void addToWriteQueue(ChunkSizedData datatype) { debugCheck(); @@ -85,68 +125,82 @@ public class DataMetaFile extends MetaFile { appendLock.unlock(); } } - private void swapWriteQueue() { - GuardedMultiAppendQueue queue = writeQueue.getAndSet(_backQueue); - // Acquire write lock and then release it again as we only need to ensure that the queue - // is not being appended to by another thread. Note that the above atomic swap & - // the guarantee that all append first acquire the appendLock means after the locK() call, - // there will be no other threads able to or is currently appending to the queue. - // Note: The above needs the getAndSet() to have at least Release Memory order. - // (not that java supports anything non volatile for getAndSet()...) - queue.appendLock.writeLock().lock(); - queue.appendLock.writeLock().unlock(); - _backQueue = queue; - } - // Load a metaFile in this path. It also automatically read the metadata. - public DataMetaFile(ILevel level, File path) throws IOException { - super(path); + // Cause: Generic Type runtime casting cannot safety check it. + // However, the Union type ensures the 'data' should only contain the listed type. + public CompletableFuture loadOrGetCached() { debugCheck(); - this.level = level; - loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion); - if (loader == null) { - throw new IOException("Invalid file: Data type loader not found: " - + dataTypeId + "(v" + loaderVersion + ")"); - } - dataType = loader.clazz; - } + Object obj = data.get(); + + CompletableFuture cached = _readCached(obj); + if (cached != null) return cached; - // Make a new MetaFile. It doesn't load or write any metadata itself. - public DataMetaFile(ILevel level, File path, DhSectionPos pos, CompletableFuture creator) { - super(path, pos); - debugCheck(); - this.level = level; CompletableFuture future = new CompletableFuture<>(); - data.set(future); - creator.thenApply((f) -> { - applyWriteQueue(f); - return f; - }).whenComplete((f, e) -> { - if (e != null) { - LOGGER.error("Uncaught error on creation {}: ", path, e); - future.complete(null); - data.set(null); - } else { - future.complete(f); - new DataObjTracker(f); - data.set(new SoftReference<>(f)); - } - }); - } - - public boolean isValid(int version) { - debugCheck(); - boolean isValid; - // First check if write queue is empty, then check if localVersion is equal to version. - // Must be done in this order as writer will increment localVersion before polling in the write queue. - // Note: Be careful with the localVerion read's memory order if we do switch over to java 1.9. - // It should be acquire or higher! - isValid = writeQueue.get().queue.isEmpty(); // The 'get()' & 'isEmpty()' enforce a memory barrier. - // Also, we are just querying the state, and this means no - // need to get any locks for the queue. - isValid &= localVersion.get() == version; // The 'get()' enforce a memory barrier. - return isValid; + // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( + boolean worked = data.compareAndSet(obj, future); + if (!worked) return loadOrGetCached(); + + // After cas. We are in exclusive control. + if (!doesFileExist) { + doesFileExist = true; + handler.onCreateDataFile(this) + .thenApply((data) -> { + metaData = makeMetaData(data); + return data; + }) + .thenApply((data) -> handler.onDataFileLoaded(data, this::applyWriteQueue)) + .whenComplete((v, e) -> { + if (e != null) { + LOGGER.error("Uncaught error on creation {}: ", path, e); + future.complete(null); + data.set(null); + } else { + future.complete(v); + new DataObjTracker(v); + data.set(new SoftReference<>(v)); + } + }); + } else { + CompletableFuture.supplyAsync(() -> { + if (metaData == null) + throw new IllegalStateException("Meta data not loaded!"); + // Load the file. + LodDataSource data; + try (FileInputStream fio = getDataContent()){ + data = loader.loadData(this, fio, level); + } catch (IOException e) { + throw new CompletionException(e); + } + // Apply the write queue + LodUtil.assertTrue(!inCacheWriteAccessAsserter.get(),"No one should be writing to the cache while we are in the process of " + + "loading one into the cache! Is this a deadlock?"); + data = handler.onDataFileLoaded(data, this::applyWriteQueue); + // Finally, return the data. + return data; + }, handler.getIOExecutor()) + .whenComplete((f, e) -> { + if (e != null) { + LOGGER.error("Error loading file {}: ", path, e); + future.complete(null); + data.set(null); + } else { + future.complete(f); + new DataObjTracker(f); + data.set(new SoftReference<>(f)); + } + }); + } + + // Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :( + //return future.completeAsync(this::loadAndUpdateDataSource, fileReaderThreads); + return future; + } + + private static MetaData makeMetaData(LodDataSource data) { + DataSourceLoader loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion()); + return new MetaData(data.getSectionPos(), -1, 1, + data.getDataDetail(), loader == null ? 0 : loader.datatypeId, data.getDataVersion()); } // "unchecked": Suppress casting of CompletableFuture to CompletableFuture @@ -169,13 +223,13 @@ public class DataMetaFile extends MetaFile { // The latter give us immediate access to the data, but we need to ensure concurrent reads and // writes doesn't cause unexpected behavior down the line. // For now, I'll go for the latter option and just hope nothing goes wrong... - if (inCacheWriteLock.getAndSet(true) == false) { + if (inCacheWriteAccessAsserter.getAndSet(true) == false) { try { - applyWriteQueue((LodDataSource) inner); + inner = handler.onDataFileRefresh((LodDataSource) inner, this::applyWriteQueue); } catch (Exception e) { LOGGER.error("Error while applying changes to LodDataSource at {}: ", pos, e); } finally { - inCacheWriteLock.set(false); + inCacheWriteAccessAsserter.set(false); } } } @@ -192,36 +246,17 @@ public class DataMetaFile extends MetaFile { return null; } - // Cause: Generic Type runtime casting cannot safety check it. - // However, the Union type ensures the 'data' should only contain the listed type. - public CompletableFuture loadOrGetCached(Executor fileReaderThreads) { - debugCheck(); - Object obj = data.get(); - - CompletableFuture cached = _readCached(obj); - if (cached != null) return cached; - - CompletableFuture future = new CompletableFuture<>(); - - // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( - boolean worked = data.compareAndSet(obj, future); - if (!worked) return loadOrGetCached(fileReaderThreads); - - // Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :( - //return future.completeAsync(this::loadAndUpdateDataSource, fileReaderThreads); - CompletableFuture.supplyAsync(this::loadAndUpdateDataSource, fileReaderThreads) - .whenComplete((f, e) -> { - if (e != null) { - LOGGER.error("Uncaught error loading file {}: ", path, e); - future.complete(null); - data.set(null); - } else { - future.complete(f); - new DataObjTracker(f); - data.set(new SoftReference<>(f)); - } - }); - return future; + private void swapWriteQueue() { + GuardedMultiAppendQueue queue = writeQueue.getAndSet(_backQueue); + // Acquire write lock and then release it again as we only need to ensure that the queue + // is not being appended to by another thread. Note that the above atomic swap & + // the guarantee that all append first acquire the appendLock means after the locK() call, + // there will be no other threads able to or is currently appending to the queue. + // Note: The above needs the getAndSet() to have at least Release Memory order. + // (not that java supports anything non volatile for getAndSet()...) + queue.appendLock.writeLock().lock(); + queue.appendLock.writeLock().unlock(); + _backQueue = queue; } // Return whether any write has happened to the data @@ -230,54 +265,30 @@ public class DataMetaFile extends MetaFile { // First check if write queue is empty, then swap the write queue. // Must be done in this order to ensure isValid work properly. See isValid() for details. boolean isEmpty = writeQueue.get().queue.isEmpty(); - int localVer; if (!isEmpty) { - localVer = localVersion.incrementAndGet(); swapWriteQueue(); int count = _backQueue.queue.size(); for (ChunkSizedData chunk : _backQueue.queue) { data.update(chunk); } _backQueue.queue.clear(); - write(data); - LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count); - } else localVer = localVersion.get(); - data.setLocalVersion(localVer); - } - - private LodDataSource loadAndUpdateDataSource() { - LodDataSource data = loadFile(); - if (data == null) data = FullDataSource.createEmpty(pos); - // Apply the write queue - LodUtil.assertTrue(!inCacheWriteLock.get(),"No one should be writing to the cache while we are in the process of " + - "loading one into the cache! Is this a deadlock?"); - applyWriteQueue(data); - // Finally, return the data. - return data; + try { + // Write/Update data + LodUtil.assertTrue(metaData != null); + metaData.dataLevel = data.getDataDetail(); + loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion()); + LodUtil.assertTrue(loader != null, "No loader for {} (v{})", data.getClass(), data.getDataVersion()); + dataType = data.getClass(); + metaData.dataTypeId = loader == null ? 0 : loader.datatypeId; + metaData.loaderVersion = data.getDataVersion(); + super.writeData((out) -> data.saveData(level, this, out)); + LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count); + } catch (IOException e) { + LOGGER.error("Failed to save updated data file at {} for sect {}", path, pos, e); + } + } } - private LodDataSource loadFile() { - if (!path.exists()) return null; - // Refresh the metadata. - try { - super.updateMetaData(); - } catch (Exception e) { - LOGGER.warn("Metadata for file {} changed unexpectedly and in an invalid state. Dropping file.", path, e); - return null; - } - if (loader == null) { - //LOGGER.warn("No loader for file {}. Dropping file.", path); // Disable as data lod has no loader yet. - return null; - } - - // Load the file. - try (FileInputStream fio = getDataContent()){ - return loader.loadData(this, fio, level); - } catch (Exception e) { - LOGGER.warn("Failed to load file {}. Dropping file.", path, e); - return null; - } - } private FileInputStream getDataContent() throws IOException { FileInputStream fin = new FileInputStream(path); int toSkip = METADATA_SIZE; @@ -295,51 +306,6 @@ public class DataMetaFile extends MetaFile { } - public CompletableFuture flushAndSave(Executor fileWriterThreads) { - debugCheck(); - boolean isEmpty = writeQueue.get().queue.isEmpty(); - if (!isEmpty) { - return loadOrGetCached(fileWriterThreads).thenApply((unused) -> null); // This will flush the data to disk. - } else { - return CompletableFuture.completedFuture(null); - } - } - - @Override - protected void updateMetaData() throws IOException { - super.updateMetaData(); - loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion); - if (loader == null) { - throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")"); - } - dataType = loader.clazz; - dataTypeId = loader.datatypeId; - } - - private void write(LodDataSource data) { - try { - dataLevel = data.getDataDetail(); - loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion()); - // FIXME: Uncomment this and fix id when we have FullDataSource loader! - //LodUtil.assertTrue(loader != null, "No loader for {} (v{})", data.getClass(), data.getDataVersion()); - dataType = data.getClass(); - dataTypeId = loader == null ? 0 : loader.datatypeId; - loaderVersion = data.getDataVersion(); - timestamp = System.currentTimeMillis(); // TODO: Do we need to use server synced time? - // Warn: This may become an attack vector! Be careful! - super.writeData((out) -> { - try { - data.saveData(level, this, out); - } catch (IOException e) { - LOGGER.error("Failed to save data for file {}", path, e); - } - }); - } catch (IOException e) { - LOGGER.error("Failed to write data for file {}", path, e); - } - } - - public static void debugCheck() { DataObjTracker phantom = (DataObjTracker) lifeCycleDebugQueue.poll(); while (phantom != null) { diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java index 56fe8b8de..c2c5de641 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java @@ -1,12 +1,149 @@ package com.seibel.lod.core.a7.save.io.file; +import com.seibel.lod.core.a7.datatype.LodDataSource; +import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; +import com.seibel.lod.core.a7.datatype.full.FullDataSource; +import com.seibel.lod.core.a7.datatype.full.SparseDataSource; import com.seibel.lod.core.a7.generation.GenerationQueue; import com.seibel.lod.core.a7.level.IServerLevel; +import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.logging.DhLoggerBuilder; +import com.seibel.lod.core.util.LodUtil; +import org.apache.logging.log4j.Logger; import java.io.File; +import java.lang.ref.WeakReference; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; public class GeneratedDataFileHandler extends DataFileHandler { - public GeneratedDataFileHandler(IServerLevel level, File saveRootDir, GenerationQueue queue) { - super(level, saveRootDir, queue::generate); + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + AtomicReference queue = new AtomicReference<>(null); + // TODO: Should I include a lib that impl weak concurrent hash map? + final Map genQueue = Collections.synchronizedMap(new WeakHashMap<>()); + + class GenTask extends GenerationQueue.GenTaskTracker { + final DhSectionPos pos; + WeakReference targetData; + LodDataSource loadedTargetData = null; + GenTask(DhSectionPos pos, WeakReference targetData) { + this.pos = pos; + this.targetData = targetData; + } + @Override + public boolean isValid() { + return targetData.get() != null; + } + @Override + public Consumer getConsumer() { + if (loadedTargetData == null) { + loadedTargetData = targetData.get(); + if (loadedTargetData == null) return null; + } + return (chunk) -> { + if (chunk.getBBoxLodPos().overlaps(loadedTargetData.getSectionPos().getSectionBBoxPos())) + write(loadedTargetData.getSectionPos(), chunk); + }; + } + + void releaseStrongReference() { + loadedTargetData = null; + } + } + + + public GeneratedDataFileHandler(IServerLevel level, File saveRootDir) { + super(level, saveRootDir); + } + + public void setGenerationQueue(GenerationQueue newQueue) { + boolean worked = queue.compareAndSet(null, newQueue); + LodUtil.assertTrue(worked, "previous queue is still here!"); + synchronized (genQueue) { + for (Map.Entry entry : genQueue.entrySet()) { + LodDataSource source = entry.getKey(); + DhSectionPos pos = source.getSectionPos(); + GenTask task = entry.getValue(); + queue.get().submitGenTask(pos.getSectionBBoxPos(), source.getDataDetail(), task) + .whenComplete( + (b, ex) -> { + if (ex != null) LOGGER.error("Uncaught Gen Task Exception at {}:", pos, ex); + LodDataSource data = task.targetData.get(); + if (ex == null && b) { + files.get(task.pos).metaData.dataVersion.incrementAndGet(); + genQueue.remove(data, task); + return; + } + task.releaseStrongReference(); + } + ); + } + } + } + + public GenerationQueue popGenerationQueue() { + GenerationQueue cas = queue.getAndSet(null); + LodUtil.assertTrue(cas != null, "there are no previous live generation queue!"); + return cas; + } + + @Override + public CompletableFuture onCreateDataFile(DataMetaFile file) { + DhSectionPos pos = file.pos; + ArrayList existFiles = new ArrayList<>(); + ArrayList missing = new ArrayList<>(); + selfSearch(pos, pos, existFiles, missing); + LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty()); + if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) { + // None exist. + SparseDataSource dataSource = SparseDataSource.createEmpty(pos); + GenerationQueue getQueue = queue.get(); + GenTask task = new GenTask(pos, new WeakReference<>(dataSource)); + genQueue.put(dataSource, task); + if (getQueue != null) { + getQueue.submitGenTask(dataSource.getSectionPos().getSectionBBoxPos(), + dataSource.getDataDetail(), task) + .whenComplete( + (b, ex) -> { + if (ex != null) LOGGER.error("Uncaught Gen Task Exception at {}:", pos, ex); + LodDataSource data = task.targetData.get(); + if (ex == null && b) { + files.get(task.pos).metaData.dataVersion.incrementAndGet(); + genQueue.remove(data, task); + return; + } + task.releaseStrongReference(); + } + ); + } + return CompletableFuture.completedFuture(dataSource); + } else { + for (DhSectionPos missingPos : missing) { + DataMetaFile newfile = atomicGetOrMakeFile(missingPos); + if (newfile != null) existFiles.add(newfile); + } + final ArrayList> futures = new ArrayList<>(existFiles.size()); + final SparseDataSource dataSource = SparseDataSource.createEmpty(pos); + + for (DataMetaFile f : existFiles) { + futures.add(f.loadOrGetCached() + .exceptionally((ex) -> null) + .thenAccept((data) -> { + if (data != null) { + if (data instanceof SparseDataSource) + dataSource.sampleFrom((SparseDataSource) data); + else if (data instanceof FullDataSource) + dataSource.sampleFrom((FullDataSource) data); + else LodUtil.assertNotReach(); + } + }) + ); + } + return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)) + .thenApply((v) -> dataSource.trySelfPromote()); + } } } diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java index 61898f88e..bc009acaf 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java @@ -3,12 +3,17 @@ package com.seibel.lod.core.a7.save.io.file; import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.full.FullFormat; +import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.a7.pos.DhSectionPos; import com.seibel.lod.core.objects.DHChunkPos; +import javax.annotation.Nullable; import java.io.File; +import java.nio.file.Path; import java.util.Collection; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.Consumer; public interface IDataSourceProvider extends AutoCloseable { void addScannedFile(Collection detectedFiles); @@ -17,5 +22,12 @@ public interface IDataSourceProvider extends AutoCloseable { void write(DhSectionPos sectionPos, ChunkSizedData chunkData); CompletableFuture flushAndSave(); - boolean isCacheValid(DhSectionPos sectionPos, long timestamp); + long getLatestCacheVersion(DhSectionPos sectionPos); + + CompletableFuture onCreateDataFile(DataMetaFile file); + LodDataSource onDataFileLoaded(LodDataSource source, Consumer updater); + LodDataSource onDataFileRefresh(LodDataSource source, Consumer updater); + File computeDataFilePath(DhSectionPos pos); + Executor getIOExecutor(); + } diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java index 43d604dcc..e05e63823 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java @@ -7,9 +7,6 @@ import java.io.File; public class RemoteDataFileHandler extends DataFileHandler { public RemoteDataFileHandler(ILevel level, File saveRootDir) { - super(level, saveRootDir, (pos) -> { - LodUtil.assertNotReach("TODO"); - return null; - }); + super(level, saveRootDir); } } diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java index 4b109eb86..115853781 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java @@ -13,4 +13,5 @@ public interface IRenderSourceProvider extends AutoCloseable { void addScannedFile(Collection detectedFiles); void write(DhSectionPos sectionPos, ChunkSizedData chunkData); CompletableFuture flushAndSave(); + boolean refreshRenderSource(LodRenderSource source); } diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java index 7d8acdc7a..4a15435e2 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java @@ -1,21 +1,26 @@ package com.seibel.lod.core.a7.save.io.render; import com.google.common.collect.HashMultimap; +import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.PlaceHolderRenderSource; import com.seibel.lod.core.a7.datatype.LodRenderSource; +import com.seibel.lod.core.a7.datatype.RenderSourceLoader; import com.seibel.lod.core.a7.datatype.column.ColumnRenderSource; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; +import com.seibel.lod.core.a7.datatype.transform.DataRenderTransformer; import com.seibel.lod.core.a7.level.IClientLevel; import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.a7.save.io.file.IDataSourceProvider; import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.a7.util.UncheckedInterruptedException; +import com.seibel.lod.core.config.Config; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; -import javax.annotation.Nullable; import java.io.File; import java.io.IOException; +import java.lang.ref.WeakReference; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -45,11 +50,7 @@ public class RenderFileHandler implements IRenderSourceProvider { { // Sort files by pos. for (File file : detectedFiles) { try { - RenderMetaFile metaFile = new RenderMetaFile( - dataSourceProvider::isCacheValid, - dataSourceProvider::read, - level, file - ); + RenderMetaFile metaFile = new RenderMetaFile(this, file); filesByPos.put(metaFile.pos, metaFile); } catch (IOException e) { throw new RuntimeException(e); @@ -57,12 +58,12 @@ public class RenderFileHandler implements IRenderSourceProvider { } } - // Warn for multiple files with the same pos, and then select the one with latest timestamp. + // Warn for multiple files with the same pos, and then select the one with the latest timestamp. for (DhSectionPos pos : filesByPos.keySet()) { Collection metaFiles = filesByPos.get(pos); RenderMetaFile fileToUse; if (metaFiles.size() > 1) { - fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.timestamp)); + fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get())); { StringBuilder sb = new StringBuilder(); sb.append("Multiple files with the same pos: "); @@ -103,19 +104,25 @@ public class RenderFileHandler implements IRenderSourceProvider { */ @Override public CompletableFuture read(DhSectionPos pos) { - RenderMetaFile metaFile = files.computeIfAbsent(pos, (p) -> new RenderMetaFile( - dataSourceProvider::isCacheValid, - dataSourceProvider::read, - level, computeDefaultFilePath(p), p)); - - return metaFile.loadOrGetCached(renderCacheThread).handle( + RenderMetaFile metaFile = files.get(pos); + if (metaFile == null) { + RenderMetaFile newMetaFile; + try { + newMetaFile = new RenderMetaFile(this, pos); + } catch (IOException e) { + LOGGER.error("IOException on creating new render file at {}", pos, e); + return null; + } + metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. + if (metaFile == null) metaFile = newMetaFile; + } + return metaFile.loadOrGetCached(renderCacheThread, level).handle( (render, e) -> { if (e != null) { LOGGER.error("Uncaught error on {}:", pos, e); } if (render != null) return render; - PlaceHolderRenderSource placeHolder = new PlaceHolderRenderSource(pos); - return placeHolder; + return new PlaceHolderRenderSource(pos); } ); } @@ -125,8 +132,8 @@ public class RenderFileHandler implements IRenderSourceProvider { */ @Override public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { - dataSourceProvider.write(sectionPos, chunkData); recursive_write(sectionPos,chunkData); + dataSourceProvider.write(sectionPos, chunkData); } private void recursive_write(DhSectionPos sectPos, ChunkSizedData chunkData) { @@ -141,7 +148,6 @@ public class RenderFileHandler implements IRenderSourceProvider { if (metaFile != null) { // Fast path: if there is a file for this section, just write to it. metaFile.updateChunkIfNeeded(chunkData); } - } /* @@ -168,4 +174,89 @@ public class RenderFileHandler implements IRenderSourceProvider { } CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } + + public File computeRenderFilePath(DhSectionPos pos) { + return new File(saveDir, pos.serialize() + ".lod"); + } + + public CompletableFuture onCreateRenderFile(RenderMetaFile file) { + final int vertSize = Config.Client.Graphics.Quality.verticalQuality + .get().calculateMaxVerticalData((byte) (file.pos.sectionDetail - ColumnRenderSource.SECTION_SIZE_OFFSET)); + return CompletableFuture.completedFuture( + new ColumnRenderSource(file.pos, vertSize, level.getMinY())); + } + + private final ConcurrentHashMap cacheRecreationGuards = new ConcurrentHashMap<>(); + + private void updateCache(LodRenderSource data, RenderMetaFile file) { + if (cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null) return; + final WeakReference dataRef = new WeakReference<>(data); + CompletableFuture dataFuture = dataSourceProvider.read(data.getSectionPos()); + final long version = dataSourceProvider.getLatestCacheVersion(data.getSectionPos()); + DataRenderTransformer.asyncTransformDataSource( + dataFuture.thenApply((d) -> { + if (dataRef.get() == null) throw new UncheckedInterruptedException(); + LodUtil.assertTrue(d != null); + return d; + }).exceptionally((ex) -> { + if (ex != null) + LOGGER.error("Uncaught exception when getting data for updateCache()", ex); + return null; + }) + , level) + .thenAccept((newData) -> write(dataRef.get(), file, newData, version)) + .exceptionally((ex) -> { + if (!UncheckedInterruptedException.isThrowableInterruption(ex)) + LOGGER.error("Exception when updating render file using data source: ", ex); + return null; + }).thenRun(() -> cacheRecreationGuards.remove(file.pos)); + + } + + public LodRenderSource onRenderFileLoaded(LodRenderSource data, RenderMetaFile file) { + long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos); + //NOTE: Do this instead of direct compare so values that wrapped around still works correctly. + if (newCacheVersion - file.metaData.dataVersion.get() <= 0) + return data; + updateCache(data, file); + return data; + } + + public LodRenderSource onLoadingRenderFile(RenderMetaFile file) { + return null; //Default behaviour + } + + private void write(LodRenderSource target, RenderMetaFile file, + LodRenderSource newData, long newDataVersion) { + if (target == null) return; + if (newData == null) return; + target.weakWrite(newData); + file.metaData.dataVersion.set(newDataVersion); + file.metaData.dataLevel = target.getDataDetail(); + file.loader = RenderSourceLoader.getLoader(target.getClass(), target.getRenderVersion()); + file.dataType = target.getClass(); + file.metaData.dataTypeId = file.loader.renderTypeId; + file.metaData.loaderVersion = target.getRenderVersion(); + file.save(target, level); + } + + public void onReadRenderSourceFromCache(RenderMetaFile file, LodRenderSource data) { + long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos); + //NOTE: Do this instead of direct compare so values that wrapped around still works correctly. + if (newCacheVersion - file.metaData.dataVersion.get() > 0) + updateCache(data, file); + } + + public boolean refreshRenderSource(LodRenderSource source) { + RenderMetaFile file = files.get(source.getSectionPos()); + LodUtil.assertTrue(file != null); + LodUtil.assertTrue(file.doesFileExist); + long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos); + //NOTE: Do this instead of direct compare so values that wrapped around still works correctly. + if (newCacheVersion - file.metaData.dataVersion.get() <= 0) + return false; + updateCache(source, file); + return true; + } + } diff --git a/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java b/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java index a747872f2..442855efc 100644 --- a/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java @@ -1,14 +1,18 @@ package com.seibel.lod.core.a7.save.io.render; +import com.seibel.lod.core.a7.datatype.DataSourceLoader; import com.seibel.lod.core.a7.datatype.LodDataSource; import com.seibel.lod.core.a7.datatype.LodRenderSource; import com.seibel.lod.core.a7.datatype.RenderSourceLoader; import com.seibel.lod.core.a7.datatype.full.ChunkSizedData; import com.seibel.lod.core.a7.datatype.transform.DataRenderTransformer; import com.seibel.lod.core.a7.level.IClientLevel; +import com.seibel.lod.core.a7.level.ILevel; import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.a7.save.io.MetaFile; import com.seibel.lod.core.a7.pos.DhSectionPos; +import com.seibel.lod.core.a7.save.io.file.DataMetaFile; +import com.seibel.lod.core.a7.save.io.file.IDataSourceProvider; import com.seibel.lod.core.util.LodUtil; import java.io.File; @@ -19,7 +23,7 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; public class RenderMetaFile extends MetaFile { - private final IClientLevel level; + //private final IClientLevel level; public RenderSourceLoader loader; public Class dataType; @@ -57,29 +61,29 @@ public class RenderMetaFile extends MetaFile { } CacheValidator validator; CacheSourceProducer source; + final RenderFileHandler handler; + boolean doesFileExist; - // Load a metaFile in this path. It also automatically read the metadata. - public RenderMetaFile(CacheValidator validator, CacheSourceProducer source, - IClientLevel level, File path) throws IOException { - super(path); - this.level = level; - loader = RenderSourceLoader.getLoader(dataTypeId, loaderVersion); - if (loader == null) { - throw new IOException("Invalid file: Data type loader not found: " - + dataTypeId + "(v" + loaderVersion + ")"); - } - dataType = loader.clazz; - this.validator = validator; - this.source = source; + + // Create a new metaFile + public RenderMetaFile(RenderFileHandler handler, DhSectionPos pos) throws IOException { + super(handler.computeRenderFilePath(pos), pos); + this.handler = handler; + LodUtil.assertTrue(metaData == null); + doesFileExist = false; } - // Make a new MetaFile. It doesn't load or write any metadata itself. - public RenderMetaFile(CacheValidator validator, CacheSourceProducer source, - IClientLevel level, File path, DhSectionPos pos) { - super(path, pos); - this.level = level; - this.validator = validator; - this.source = source; + public RenderMetaFile(RenderFileHandler handler, File path) throws IOException { + super(path); + this.handler = handler; + LodUtil.assertTrue(metaData != null); + loader = RenderSourceLoader.getLoader(metaData.dataTypeId, metaData.loaderVersion); + if (loader == null) { + throw new IOException("Invalid file: Data type loader not found: " + + metaData.dataTypeId + "(v" + metaData.loaderVersion + ")"); + } + dataType = loader.clazz; + doesFileExist = true; } // Suppress casting of CompletableFuture to CompletableFuture @@ -90,6 +94,7 @@ public class RenderMetaFile extends MetaFile { Object inner = ((SoftReference)obj).get(); if (inner != null) { LodUtil.assertTrue(inner instanceof LodRenderSource); + handler.onReadRenderSourceFromCache(this, (LodRenderSource) inner); return CompletableFuture.completedFuture((LodRenderSource)inner); } } @@ -104,7 +109,7 @@ public class RenderMetaFile extends MetaFile { // Cause: Generic Type runtime casting cannot safety check it. // However, the Union type ensures the 'data' should only contain the listed type. - public CompletableFuture loadOrGetCached(Executor fileReaderThreads) { + public CompletableFuture loadOrGetCached(Executor fileReaderThreads, ILevel level) { Object obj = data.get(); CompletableFuture cached = _readCached(obj); @@ -117,49 +122,68 @@ public class RenderMetaFile extends MetaFile { // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :( boolean worked = data.compareAndSet(obj, future); - if (!worked) return loadOrGetCached(fileReaderThreads); + if (!worked) return loadOrGetCached(fileReaderThreads, level); // Now, there should only ever be one thread at a time here due to the CAS operation above. - // Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :( - //return future.completeAsync(this::loadAndUpdateRenderSource, fileReaderThreads); - CompletableFuture.supplyAsync(() -> buildFuture(fileReaderThreads), fileReaderThreads) - .thenCompose((sourceCompletableFuture) -> sourceCompletableFuture) - .whenComplete((renderSource, e) -> { - if (e != null) { - LOGGER.error("Uncaught error loading file {}: ", path, e); - future.complete(null); - data.set(null); - } else { - future.complete(renderSource); - data.set(new SoftReference<>(renderSource)); - } - }); + + // After cas. We are in exclusive control. + if (!doesFileExist) { + doesFileExist = true; + handler.onCreateRenderFile(this) + .thenApply((data) -> { + metaData = makeMetaData(data); + return data; + }) + .thenApply((d) -> handler.onRenderFileLoaded(d, this)) + .whenComplete((v, e) -> { + if (e != null) { + LOGGER.error("Uncaught error on creation {}: ", path, e); + future.complete(null); + data.set(null); + } else { + future.complete(v); + //new DataObjTracker(v); //TODO: Obj Tracker??? For debug? + data.set(new SoftReference<>(v)); + } + }); + + + } else { + CompletableFuture.supplyAsync(() -> { + if (metaData == null) + throw new IllegalStateException("Meta data not loaded!"); + // Load the file. + LodRenderSource data; + data = handler.onLoadingRenderFile(this); + if (data == null) { + try (FileInputStream fio = getDataContent()) { + data = loader.loadRender(this, fio, level); + } catch (IOException e) { + throw new CompletionException(e); + } + } + data = handler.onRenderFileLoaded(data, this); + return data; + }, fileReaderThreads) + .whenComplete((f, e) -> { + if (e != null) { + LOGGER.error("Error loading file {}: ", path, e); + future.complete(null); + data.set(null); + } else { + future.complete(f); + data.set(new SoftReference<>(f)); + } + }); + } return future; } - private CompletableFuture buildFuture(Executor executorService) { - if (path.exists()) { - try { - updateMetaData(); - if (validator.isCacheValid(pos, timestamp)) { - // Load the file. - try (FileInputStream fio = getDataContent()) { - return CompletableFuture.completedFuture( - loader.loadRender(this, fio, level)); - } - } - } catch (IOException e) { - LOGGER.warn("Failed to read render cache at {}:", path, e); - LOGGER.warn("Will delete cache file."); - path.delete(); - } - } - // Otherwise, re-query and make the RenderSource - CompletableFuture dataFuture = source.getSourceFuture(pos); - return dataFuture.thenCombineAsync( - DataRenderTransformer.asyncTransformDataSource(dataFuture, level), - this::write, executorService); + private static MetaData makeMetaData(LodRenderSource data) { + RenderSourceLoader loader = RenderSourceLoader.getLoader(data.getClass(), data.getRenderVersion()); + return new MetaData(data.getSectionPos(), -1, -1, + data.getDataDetail(), loader == null ? 0 : loader.renderTypeId, data.getRenderVersion()); } private FileInputStream getDataContent() throws IOException { @@ -178,36 +202,13 @@ public class RenderMetaFile extends MetaFile { return fin; } - @Override - protected void updateMetaData() throws IOException { - super.updateMetaData(); - loader = RenderSourceLoader.getLoader(dataTypeId, loaderVersion); - if (loader == null) { - throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")"); - } - dataType = loader.clazz; - dataTypeId = loader.renderTypeId; - } - - private LodRenderSource write(LodDataSource parent, LodRenderSource render) { - if (parent == null) return null; + public void save(LodRenderSource data, IClientLevel level) { + LodUtil.assertTrue(data == _readCached(this.data.get()).getNow(null)); + LOGGER.info("Saving updated render file v[{}] at sect {}", metaData.dataVersion.get(), pos); try { - //TODO: Update Timestamp & stuff based on parent - dataLevel = parent.getDataDetail(); - loader = RenderSourceLoader.getLoader(render.getClass(), render.getRenderVersion()); - dataType = render.getClass(); - dataTypeId = loader.renderTypeId; - loaderVersion = render.getRenderVersion(); - super.writeData((out) -> { - try { - render.saveRender(level, this, out); - } catch (IOException e) { - LOGGER.error("Failed to save data for file {}", path, e); - } - }); + super.writeData((out) -> data.saveRender(level, this, out)); } catch (IOException e) { - LOGGER.error("Failed to write data for file {}", path, e); + LOGGER.error("Failed to save updated render file at {} for sect {}", path, pos, e); } - return render; } } diff --git a/core/src/main/java/com/seibel/lod/core/builders/worldGeneration/BatchGenerator.java b/core/src/main/java/com/seibel/lod/core/builders/worldGeneration/BatchGenerator.java index 32f098cd9..8a5f37185 100644 --- a/core/src/main/java/com/seibel/lod/core/builders/worldGeneration/BatchGenerator.java +++ b/core/src/main/java/com/seibel/lod/core/builders/worldGeneration/BatchGenerator.java @@ -40,6 +40,7 @@ import org.apache.logging.log4j.Logger; import java.lang.invoke.MethodHandles; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; public class BatchGenerator implements IChunkGenerator { @@ -264,7 +265,7 @@ public class BatchGenerator implements IChunkGenerator } @Override - public CompletableFuture> generateChunks(DHChunkPos chunkPosMin, byte granularity) { + public CompletableFuture generateChunks(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer resultConsumer) { EDistanceGenerationMode mode = Config.Client.WorldGenerator.distanceGenerationMode.get(); Steps targetStep = null; switch (mode) { @@ -286,17 +287,26 @@ public class BatchGenerator implements IChunkGenerator break; }; - int chunkXMin = chunkPosMin.x; int chunkZMin = chunkPosMin.z; int genChunkSize = 1 << (granularity - 4); // minus 4 for chunk size as its equal to div by 16 double runTimeRatio = Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get()>1 ? 1.0 : Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get(); - return generationGroup.generateChunks(chunkXMin, chunkZMin, genChunkSize, targetStep, runTimeRatio); + return generationGroup.generateChunks(chunkXMin, chunkZMin, genChunkSize, targetStep, runTimeRatio, resultConsumer); } @Override - public byte getDataDetail() { + public byte getMinDataDetail() { + return 0; + } + + @Override + public byte getMaxDataDetail() { + return 0; + } + + @Override + public int getPriority() { return 0; } @@ -307,7 +317,7 @@ public class BatchGenerator implements IChunkGenerator @Override public byte getMaxGenerationGranularity() { - return 8; + return 6; } @Override @@ -318,5 +328,4 @@ public class BatchGenerator implements IChunkGenerator public void update() { generationGroup.updateAllFutures(); } - } diff --git a/core/src/main/java/com/seibel/lod/core/util/Atomics.java b/core/src/main/java/com/seibel/lod/core/util/Atomics.java index a616ea5b1..fc38bcaa4 100644 --- a/core/src/main/java/com/seibel/lod/core/util/Atomics.java +++ b/core/src/main/java/com/seibel/lod/core/util/Atomics.java @@ -4,6 +4,7 @@ import it.unimi.dsi.fastutil.booleans.BooleanObjectImmutablePair; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.function.Predicate; public class Atomics { // While java 8 does have built in atomic operations, there doesn't seem to be any Compare And Exchange operation... @@ -26,6 +27,23 @@ public class Atomics { } } + public static T conditionalAndExchange(AtomicReference atomic, Predicate requirement, T newValue) { + while (true) { + T oldValue = atomic.get(); + if (!requirement.test(oldValue)) return oldValue; + if (atomic.weakCompareAndSet(oldValue, newValue)) return oldValue; + } + } + + public static BooleanObjectImmutablePair conditionalAndExchangeWeak(AtomicReference atomic, Predicate requirement, T newValue) { + T oldValue = atomic.get(); + if (requirement.test(oldValue) && atomic.weakCompareAndSet(oldValue, newValue)) { + return new BooleanObjectImmutablePair<>(true, oldValue); + } else { + return new BooleanObjectImmutablePair<>(false, oldValue); + } + } + // Additionally, we implement some helper methods for frequently used atomic operations. // Compare with expected value and set new value if equal. Then return whatever value the atomic now contains. diff --git a/core/src/main/java/com/seibel/lod/core/util/LodUtil.java b/core/src/main/java/com/seibel/lod/core/util/LodUtil.java index 74eaf90f3..fddabe10f 100644 --- a/core/src/main/java/com/seibel/lod/core/util/LodUtil.java +++ b/core/src/main/java/com/seibel/lod/core/util/LodUtil.java @@ -432,6 +432,10 @@ public class LodUtil public static void assertNotReach(String message, Object... args) { throw new AssertFailureException("Assert Not Reach failed:\n " + formatLog(message, args)); } + public static void assertToDo() { + throw new AssertFailureException("TODO!"); + } + public static ExecutorService makeSingleThreadPool(String name, int relativePriority) { return Executors.newFixedThreadPool(1, new LodThreadFactory(name, Thread.NORM_PRIORITY+relativePriority)); } diff --git a/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvionmentWrapper.java b/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvionmentWrapper.java index 806e29ff6..bda96e1dd 100644 --- a/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvionmentWrapper.java +++ b/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvionmentWrapper.java @@ -20,10 +20,10 @@ package com.seibel.lod.core.wrapperInterfaces.worldGeneration; import com.seibel.lod.core.a7.level.ILevel; -import com.seibel.lod.core.util.gridList.ArrayGridList; import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; public abstract class AbstractBatchGenerationEnvionmentWrapper { public enum Steps { @@ -41,5 +41,5 @@ public abstract class AbstractBatchGenerationEnvionmentWrapper { public abstract void stop(boolean blocking); - public abstract CompletableFuture> generateChunks(int minX, int minZ, int genSize, Steps targetStep, double runTimeRatio); + public abstract CompletableFuture generateChunks(int minX, int minZ, int genSize, Steps targetStep, double runTimeRatio, Consumer resultConsumer); }