From b4ea8854a835da4ab35dfd832364bfa9d34d761e Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Sun, 18 Sep 2022 16:29:59 +0800 Subject: [PATCH 1/3] Fix critical issue causing chunk to lod build extremely slow, and also partly fix sparse data source loading (where I used wrong array instead of loaded data array.) Also improve ChunkToLodBuilder building loops to support multithreaded building --- .../core/datatype/full/SparseDataSource.java | 2 +- .../datatype/transform/ChunkToLodBuilder.java | 69 ++++++++++++------- .../datatype/transform/LodDataBuilder.java | 1 + .../file/renderfile/RenderFileHandler.java | 4 ++ .../lod/core/util/objects/EventLoop.java | 1 + 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java b/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java index 385c62d8a..54cedfade 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java @@ -280,7 +280,7 @@ public class SparseDataSource implements LodDataSource { FullArrayView[] objectChunks = new FullArrayView[chunks*chunks]; for (int i=0; i latestChunkToBuild = new ConcurrentHashMap<>(); private final ConcurrentLinkedDeque taskToBuild = new ConcurrentLinkedDeque<>(); - private final ExecutorService executor = LodUtil.makeSingleThreadPool(ChunkToLodBuilder.class); - private final EventLoop ticker = new EventLoop(executor, this::_tick); + private final ExecutorService executor = LodUtil.makeThreadPool(4, ChunkToLodBuilder.class); + private final AtomicInteger runningCount = new AtomicInteger(0); public CompletableFuture tryGenerateData(IChunkWrapper chunk) { if (chunk == null) throw new NullPointerException("ChunkWrapper cannot be null!"); @@ -43,33 +45,48 @@ public class ChunkToLodBuilder { } public void tick() { - ticker.tick(); + while (true) { + if (runningCount.get() > 8192) return; + Task task = taskToBuild.pollFirst(); + if (task == null) return; // There's no jobs. + IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation + if (latestChunk == null) { + LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos); + task.future.complete(null); + return; + } + + runningCount.incrementAndGet(); + CompletableFuture.supplyAsync(() -> { + long time = System.nanoTime(); + if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { + ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk); + if (data != null) { + long time2 = System.nanoTime(); + LOGGER.info("Processed Task at {} using {}", task.chunkPos, Duration.ofNanos(time2 - time)); + task.future.complete(data); + return true; + } + } + return false; + }, executor).handle((b, ex) -> { + runningCount.decrementAndGet(); + if (ex == null && b) return true; + if (ex != null) { + LOGGER.error("Error while processing Task at {}!", task.chunkPos, ex); + } + // Failed to build due to chunk not meeting requirement. + IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null + if (casChunk == null) // That means CAS have been successful + taskToBuild.addLast(task); // Then add back the same old task. + else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. + task.future.complete(null); + return false; + }); + } } private void _tick() { - Task task = taskToBuild.pollFirst(); - if (task == null) return; // There's no jobs. - IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation - if (latestChunk == null) { - LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos); - task.future.complete(null); - return; - } - - if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { - ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk); - if (data != null) { - task.future.complete(data); - return; - } - } - - // Failed to build due to chunk not meeting requirement. - IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null - if (casChunk == null) // That means CAS have been successful - taskToBuild.addLast(task); // Then add back the same old task. - else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. - task.future.complete(null); } } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java b/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java index 67acc4d35..e94d1908a 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java @@ -58,6 +58,7 @@ public class LodDataBuilder { public static boolean canGenerateLodFromChunk(IChunkWrapper chunk) { + //return true; return chunk != null && chunk.isLightCorrect(); } diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java index f49d06700..258870079 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java @@ -140,6 +140,10 @@ public class RenderFileHandler implements IRenderSourceProvider { */ @Override public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { + if (chunkData.getBBoxLodPos().convertUpwardsTo((byte)6).equals(new DhLodPos((byte)6, 10, -11))) { + int doNothing = 0; + } + recursive_write(sectionPos,chunkData); dataSourceProvider.write(sectionPos, chunkData); } diff --git a/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java b/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java index b0e63dc2c..0d202656b 100644 --- a/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java +++ b/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java @@ -40,6 +40,7 @@ public class EventLoop implements AutoCloseable { future.cancel(true); } future = null; + executorService.shutdown(); } public boolean isRunning() { return future != null && !future.isDone(); From 41585731298994b9149c2878c22e4c7e6337e060 Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Mon, 19 Sep 2022 12:28:39 +0800 Subject: [PATCH 2/3] Add render source flag debug mode & fix sparse source to render source transform bug --- .../lod/api/enums/rendering/EDebugMode.java | 18 +++++++++-- .../datatype/column/ColumnRenderSource.java | 30 +++++++++++++++++++ .../column/render/ColumnRenderBuffer.java | 3 +- .../column/render/CubicLodTemplate.java | 11 ++++++- .../datatype/full/IdBiomeBlockStateMap.java | 14 ++++----- .../core/datatype/full/SparseDataSource.java | 7 ++--- .../transform/FullToColumnTransformer.java | 4 +++ .../seibel/lod/core/render/LodQuadTree.java | 2 +- .../lod/core/render/renderer/LodRenderer.java | 3 +- .../com/seibel/lod/core/util/ColorUtil.java | 13 +++++++- 10 files changed, 85 insertions(+), 20 deletions(-) diff --git a/api/src/main/java/com/seibel/lod/api/enums/rendering/EDebugMode.java b/api/src/main/java/com/seibel/lod/api/enums/rendering/EDebugMode.java index 2ac01c6a2..ac30a42c9 100644 --- a/api/src/main/java/com/seibel/lod/api/enums/rendering/EDebugMode.java +++ b/api/src/main/java/com/seibel/lod/api/enums/rendering/EDebugMode.java @@ -28,6 +28,8 @@ package com.seibel.lod.api.enums.rendering; * SHOW_GENMODE_WIREFRAME,
* SHOW_OVERLAPPING_QUADS,
* SHOW_OVERLAPPING_QUADS_WIREFRAME,
+ * SHOW_RENDER_SOURCE_FLAG,
+ * SHOW_RENDER_SOURCE_FLAG_WIREFRAME,
* * @author Leetom * @author James Seibel @@ -62,8 +64,14 @@ public enum EDebugMode SHOW_OVERLAPPING_QUADS, /** Only draw overlapping LOD quads, and draws in wireframe. */ - SHOW_OVERLAPPING_QUADS_WIREFRAME; - + SHOW_OVERLAPPING_QUADS_WIREFRAME, + + /** LOD colors are based on renderSource flags. */ + SHOW_RENDER_SOURCE_FLAG, + + /** LOD colors are based on renderSource flags, and draws in wireframe. */ + SHOW_RENDER_SOURCE_FLAG_WIREFRAME; + /** returns the next debug mode */ // Deprecated: use DebugMode.next() instead @Deprecated @@ -81,13 +89,17 @@ public enum EDebugMode case SHOW_GENMODE: return SHOW_GENMODE_WIREFRAME; case SHOW_GENMODE_WIREFRAME: return SHOW_OVERLAPPING_QUADS; case SHOW_OVERLAPPING_QUADS: return SHOW_OVERLAPPING_QUADS_WIREFRAME; + case SHOW_OVERLAPPING_QUADS_WIREFRAME: return SHOW_RENDER_SOURCE_FLAG; + case SHOW_RENDER_SOURCE_FLAG: return SHOW_RENDER_SOURCE_FLAG_WIREFRAME; default: return OFF; } } public static EDebugMode previous(EDebugMode type) { switch (type) { - case OFF: return SHOW_OVERLAPPING_QUADS_WIREFRAME; + case OFF: return SHOW_RENDER_SOURCE_FLAG_WIREFRAME; + case SHOW_RENDER_SOURCE_FLAG_WIREFRAME: return SHOW_RENDER_SOURCE_FLAG; + case SHOW_RENDER_SOURCE_FLAG: return SHOW_OVERLAPPING_QUADS_WIREFRAME; case SHOW_OVERLAPPING_QUADS_WIREFRAME: return SHOW_OVERLAPPING_QUADS; case SHOW_OVERLAPPING_QUADS: return SHOW_GENMODE_WIREFRAME; case SHOW_GENMODE_WIREFRAME: return SHOW_GENMODE; diff --git a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java index 63b62fd5a..9c27831b7 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/column/ColumnRenderSource.java @@ -15,6 +15,7 @@ import com.seibel.lod.core.level.ILevel; import com.seibel.lod.core.render.LodQuadTree; import com.seibel.lod.core.render.LodRenderSection; import com.seibel.lod.core.datatype.LodRenderSource; +import com.seibel.lod.core.util.ColorUtil; import com.seibel.lod.core.util.objects.Reference; import com.seibel.lod.core.util.LodUtil; import org.apache.logging.log4j.Logger; @@ -45,6 +46,31 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype { public final long[] dataContainer; public final int[] airDataContainer; + public enum DebugSourceFlag { + FULL(ColorUtil.BLUE), + DIRECT(ColorUtil.WHITE), + SPARSE(ColorUtil.YELLOW), + FILE(ColorUtil.BROWN); + public final int color; + DebugSourceFlag(int color) { + this.color = color; + } + } + + public final DebugSourceFlag[] debugSourceFlags; + + public void debugFillFlag(int ox, int oz, int w, int h, DebugSourceFlag flag) { + for (int x = ox; x < ox + w; x++) { + for (int z = oz; z < oz + h; z++) { + debugSourceFlags[x * SECTION_SIZE + z] = flag; + } + } + } + + public DebugSourceFlag debugGetFlag(int ox, int oz) { + return debugSourceFlags[ox * SECTION_SIZE + oz]; + } + private boolean isEmpty = true; /** @@ -55,6 +81,7 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype { verticalSize = maxVerticalSize; dataContainer = new long[SECTION_SIZE * SECTION_SIZE * verticalSize]; airDataContainer = new int[AIR_SECTION_SIZE * AIR_SECTION_SIZE * verticalSize]; + debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; this.sectionPos = sectionPos; this.yOffset = yOffset; } @@ -98,6 +125,8 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype { verticalSize = inputData.readByte() & 0b01111111; dataContainer = loadData(inputData, version, verticalSize); airDataContainer = new int[AIR_SECTION_SIZE * AIR_SECTION_SIZE * verticalSize]; + debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; + debugFillFlag(0, 0, SECTION_SIZE, SECTION_SIZE, DebugSourceFlag.FILE); } @Override @@ -381,6 +410,7 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype { if (genMode <= srcGenMode) { new ColumnArrayView(dataContainer, verticalSize, i, verticalSize).copyFrom( new ColumnArrayView(src.dataContainer, verticalSize, i, verticalSize)); + debugSourceFlags[i/verticalSize] = src.debugSourceFlags[i/verticalSize]; } } } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/column/render/ColumnRenderBuffer.java b/core/src/main/java/com/seibel/lod/core/datatype/column/render/ColumnRenderBuffer.java index a7813d2f0..29a44ea3e 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/column/render/ColumnRenderBuffer.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/column/render/ColumnRenderBuffer.java @@ -302,6 +302,7 @@ public class ColumnRenderBuffer extends RenderBuffer { if (posData.size() == 0 || !ColumnFormat.doesItExist(posData.get(0)) || ColumnFormat.isVoid(posData.get(0))) continue; + ColumnRenderSource.DebugSourceFlag debugSourceFlag = region.debugGetFlag(x, z); ColumnArrayView[][] adjData = new ColumnArrayView[4][]; // We extract the adj data in the four cardinal direction @@ -381,7 +382,7 @@ public class ColumnRenderBuffer extends RenderBuffer { long adjDataBot = i + 1 < posData.size() ? posData.get(i + 1) : ColumnFormat.EMPTY_DATA; CubicLodTemplate.addLodToBuffer(data, adjDataTop, adjDataBot, adjData, detailLevel, - x, z, quadBuilder, debugMode); + x, z, quadBuilder, debugMode, debugSourceFlag); } } } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/column/render/CubicLodTemplate.java b/core/src/main/java/com/seibel/lod/core/datatype/column/render/CubicLodTemplate.java index b1246799c..307170139 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/column/render/CubicLodTemplate.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/column/render/CubicLodTemplate.java @@ -19,6 +19,7 @@ package com.seibel.lod.core.datatype.column.render; +import com.seibel.lod.core.datatype.column.ColumnRenderSource; import com.seibel.lod.core.datatype.column.accessor.ColumnFormat; import com.seibel.lod.api.enums.rendering.EDebugMode; import com.seibel.lod.core.dependencyInjection.SingletonInjector; @@ -38,7 +39,8 @@ public class CubicLodTemplate private static final ILodConfigWrapperSingleton CONFIG = SingletonInjector.INSTANCE.get(ILodConfigWrapperSingleton.class); - public static void addLodToBuffer(long data, long topData, long botData, ColumnArrayView[][] adjData, byte detailLevel, int offsetPosX, int offsetOosZ, LodQuadBuilder quadBuilder, EDebugMode debugging) + public static void addLodToBuffer(long data, long topData, long botData, ColumnArrayView[][] adjData, + byte detailLevel, int offsetPosX, int offsetOosZ, LodQuadBuilder quadBuilder, EDebugMode debugging, ColumnRenderSource.DebugSourceFlag debugSource) { short width = (short) (1 << detailLevel); short x = (short) LevelPosUtil.convert(detailLevel, offsetPosX, LodUtil.BLOCK_DETAIL_LEVEL); @@ -92,6 +94,13 @@ public class CubicLodTemplate fullBright = true; break; } + case SHOW_RENDER_SOURCE_FLAG: + case SHOW_RENDER_SOURCE_FLAG_WIREFRAME: + { + color = debugSource == null ? ColorUtil.RED : debugSource.color; + fullBright = true; + break; + } default: throw new IllegalArgumentException("Unknown debug mode: " + debugging); } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/full/IdBiomeBlockStateMap.java b/core/src/main/java/com/seibel/lod/core/datatype/full/IdBiomeBlockStateMap.java index 03302a5f0..0179cb3cf 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/full/IdBiomeBlockStateMap.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/full/IdBiomeBlockStateMap.java @@ -98,12 +98,12 @@ public class IdBiomeBlockStateMap { @Override public boolean equals(Object other) { if (other == this) return true; - if (!(other instanceof IdBiomeBlockStateMap)) return false; - IdBiomeBlockStateMap otherMap = (IdBiomeBlockStateMap) other; - if (entries.size() != otherMap.entries.size()) return false; - for (int i=0; i chunks*chunks/8+64) + if (length < 0 || length > (chunks*chunks/8+64)*2) throw new IOException(LodUtil.formatLog("Sparse Flag BitSet size outside reasonable range: {} (expects {} to {})", length, 1, chunks*chunks/8+63)); byte[] bytes = dos.readNBytes(length); BitSet set = BitSet.valueOf(bytes); - if (set.size() < chunks*chunks) - throw new IOException((LodUtil.formatLog("Sparse Flag BitSet too small: {} != {}*{}", - set.size(), chunks, chunks))); long[][][] dataChunks = new long[chunks*chunks][][]; @@ -332,6 +329,6 @@ public class SparseDataSource implements LodDataSource { int chunkZ = z / dataPerChunk; FullArrayView chunk = sparseData[chunkX * chunks + chunkZ]; if (chunk == null) return null; - return chunk.get(chunkX % dataPerChunk, chunkZ % dataPerChunk); + return chunk.get(x % dataPerChunk, z % dataPerChunk); } } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java b/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java index 289fc82e3..eb3d2478b 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/transform/FullToColumnTransformer.java @@ -45,6 +45,7 @@ public class FullToColumnTransformer { if (fullArrayView.doesItExist()) LodUtil.assertTrue(columnSource.doesItExist(x, z)); } } + columnSource.debugFillFlag(0, 0, ColumnRenderSource.SECTION_SIZE, ColumnRenderSource.SECTION_SIZE, ColumnRenderSource.DebugSourceFlag.FULL); // } else if (dataDetail == 0 && columnSource.getDataDetail() > dataDetail) { // byte deltaDetail = (byte) (columnSource.getDataDetail() - dataDetail); // int perColumnWidth = 1 << deltaDetail; @@ -82,6 +83,7 @@ public class FullToColumnTransformer { if (fullArrayView == null) continue; ColumnArrayView columnArrayView = columnSource.getVerticalDataView(x, z); convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView, 1); + columnSource.debugFillFlag(x, z, 1, 1, ColumnRenderSource.DebugSourceFlag.SPARSE); if (fullArrayView.doesItExist()) LodUtil.assertTrue(columnSource.doesItExist(x, z)); } } @@ -117,6 +119,7 @@ public class FullToColumnTransformer { if (fullArrayView.doesItExist()) LodUtil.assertTrue(render.doesItExist(renderOffsetX + x, renderOffsetZ + z)); } } + render.debugFillFlag(renderOffsetX, renderOffsetZ, 16, 16, ColumnRenderSource.DebugSourceFlag.DIRECT); } else { final int dataPerRender = 1 << (render.getDataDetail() - data.dataDetail); final int dataSize = 16 / dataPerRender; @@ -141,6 +144,7 @@ public class FullToColumnTransformer { downSampledArrayView.mergeMultiDataFrom(tempQuadView); } } + render.debugFillFlag(renderOffsetX, renderOffsetZ, dataSize, dataSize, ColumnRenderSource.DebugSourceFlag.DIRECT); } } diff --git a/core/src/main/java/com/seibel/lod/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/lod/core/render/LodQuadTree.java index ba74c246b..ce4076ac8 100644 --- a/core/src/main/java/com/seibel/lod/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/lod/core/render/LodQuadTree.java @@ -78,7 +78,7 @@ public class LodQuadTree implements AutoCloseable { for (byte i = LAYER_BEGINNING_OFFSET; i < numbersOfSectionLevels; i++) { byte targetDataDetail = getLayerDataDetail(i); int maxDist = getFurthestDistance(targetDataDetail); - int halfSize = MathUtil.ceilDiv(maxDist, (1 << i)) + 2; // +2 to make sure the section is fully contained in the ringList + int halfSize = MathUtil.ceilDiv(maxDist, (1 << i)) + 8; // +8 to make sure the section is fully contained in the ringList { DhSectionPos checkerPos = new DhSectionPos(i, halfSize, halfSize); byte checkedDetail = calculateExpectedDetailLevel(new DhBlockPos2D(initialPlayerX, initialPlayerZ),checkerPos); diff --git a/core/src/main/java/com/seibel/lod/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/lod/core/render/renderer/LodRenderer.java index 1c1943747..c55afd193 100644 --- a/core/src/main/java/com/seibel/lod/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/lod/core/render/renderer/LodRenderer.java @@ -163,7 +163,8 @@ public class LodRenderer if (debugModeConfig.get() == EDebugMode.SHOW_DETAIL_WIREFRAME || debugModeConfig.get() == EDebugMode.SHOW_GENMODE_WIREFRAME || debugModeConfig.get() == EDebugMode.SHOW_WIREFRAME - || debugModeConfig.get() == EDebugMode.SHOW_OVERLAPPING_QUADS_WIREFRAME) { + || debugModeConfig.get() == EDebugMode.SHOW_OVERLAPPING_QUADS_WIREFRAME + || debugModeConfig.get() == EDebugMode.SHOW_RENDER_SOURCE_FLAG_WIREFRAME) { GL32.glPolygonMode(GL32.GL_FRONT_AND_BACK, GL32.GL_LINE); //GL32.glDisable(GL32.GL_CULL_FACE); } diff --git a/core/src/main/java/com/seibel/lod/core/util/ColorUtil.java b/core/src/main/java/com/seibel/lod/core/util/ColorUtil.java index ecf07c627..44665731c 100644 --- a/core/src/main/java/com/seibel/lod/core/util/ColorUtil.java +++ b/core/src/main/java/com/seibel/lod/core/util/ColorUtil.java @@ -35,8 +35,19 @@ public class ColorUtil public static final int BLACK = rgbToInt(0,0,0); public static final int WHITE = rgbToInt(255,255,255); public static final int TRANSPARENT = rgbToInt(0, 0, 0, 0); - public static final int RED = rgbToInt(255,0,0); + public static final int GREEN = rgbToInt(0,255,0); + public static final int BLUE = rgbToInt(0,0,255); + public static final int YELLOW = rgbToInt(255,255,0); + public static final int CYAN = rgbToInt(0,255,255); + public static final int MAGENTA = rgbToInt(255,0,255); + public static final int ORANGE = rgbToInt(255,128,0); + public static final int PINK = rgbToInt(255,128,128); + public static final int GRAY = rgbToInt(128,128,128); + public static final int LIGHT_GRAY = rgbToInt(192,192,192); + public static final int DARK_GRAY = rgbToInt(64,64,64); + public static final int BROWN = rgbToInt(128,64,0); + public static final int PURPLE = rgbToInt(128,0,128); public static int rgbToInt(int red, int green, int blue) { From 905e73fd1c8101c1b01f546543724ba1a59e96df Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Mon, 19 Sep 2022 14:54:45 +0800 Subject: [PATCH 3/3] Continue improving stabilises of the system --- .../datatype/transform/ChunkToLodBuilder.java | 96 +++++++++++-------- .../transform/DataRenderTransformer.java | 2 +- .../core/file/datafile/DataFileHandler.java | 5 +- .../lod/core/file/datafile/DataMetaFile.java | 4 +- .../file/datafile/IDataSourceProvider.java | 3 +- .../chunk/IChunkWrapper.java | 2 + 6 files changed, 67 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/datatype/transform/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/lod/core/datatype/transform/ChunkToLodBuilder.java index d97258855..0b0987fc0 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/transform/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/transform/ChunkToLodBuilder.java @@ -5,12 +5,10 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import com.seibel.lod.core.datatype.full.ChunkSizedData; -import com.seibel.lod.core.level.ILevel; import com.seibel.lod.core.config.Config; import com.seibel.lod.core.logging.ConfigBasedLogger; import com.seibel.lod.core.pos.DhChunkPos; import com.seibel.lod.core.util.*; -import com.seibel.lod.core.util.objects.EventLoop; import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import org.apache.logging.log4j.LogManager; @@ -18,6 +16,9 @@ import org.apache.logging.log4j.LogManager; public class ChunkToLodBuilder { public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(), () -> Config.Client.Advanced.Debugging.DebugSwitch.logLodBuilderEvent.get()); + public static final long MAX_TICK_TIME_NS = 1000000000L / 20L; + public static final int THREAD_COUNT = 1; + static class Task { final DhChunkPos chunkPos; final CompletableFuture future; @@ -28,7 +29,7 @@ public class ChunkToLodBuilder { } private final ConcurrentHashMap latestChunkToBuild = new ConcurrentHashMap<>(); private final ConcurrentLinkedDeque taskToBuild = new ConcurrentLinkedDeque<>(); - private final ExecutorService executor = LodUtil.makeThreadPool(4, ChunkToLodBuilder.class); + private final ExecutorService executor = LodUtil.makeThreadPool(THREAD_COUNT, ChunkToLodBuilder.class); private final AtomicInteger runningCount = new AtomicInteger(0); public CompletableFuture tryGenerateData(IChunkWrapper chunk) { @@ -45,48 +46,63 @@ public class ChunkToLodBuilder { } public void tick() { - while (true) { - if (runningCount.get() > 8192) return; - Task task = taskToBuild.pollFirst(); - if (task == null) return; // There's no jobs. - IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation - if (latestChunk == null) { - LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos); - task.future.complete(null); - return; - } - + if (runningCount.get() >= THREAD_COUNT) return; + if (taskToBuild.isEmpty()) return; + for (int i = 0; i { - long time = System.nanoTime(); - if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { - ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk); - if (data != null) { - long time2 = System.nanoTime(); - LOGGER.info("Processed Task at {} using {}", task.chunkPos, Duration.ofNanos(time2 - time)); - task.future.complete(data); - return true; - } + CompletableFuture.runAsync(() -> { + try { + _tick(); + } finally { + runningCount.decrementAndGet(); } - return false; - }, executor).handle((b, ex) -> { - runningCount.decrementAndGet(); - if (ex == null && b) return true; - if (ex != null) { - LOGGER.error("Error while processing Task at {}!", task.chunkPos, ex); - } - // Failed to build due to chunk not meeting requirement. - IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null - if (casChunk == null) // That means CAS have been successful - taskToBuild.addLast(task); // Then add back the same old task. - else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. - task.future.complete(null); - return false; - }); + }, executor); } } private void _tick() { - } + long time = System.nanoTime(); + int count = 0; + boolean allDone = false; + while (true) { + if (System.nanoTime() - time > MAX_TICK_TIME_NS && !taskToBuild.isEmpty()) break; + Task task = taskToBuild.pollFirst(); + if (task == null) { + allDone = true; + break; + } + count++; + IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation + if (latestChunk == null) { + LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos); + task.future.complete(null); + continue; + } + try { + if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { + ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk); + if (data != null) { + task.future.complete(data); + continue; + } + } + } catch (Exception ex) { + LOGGER.error("Error while processing Task at {}!", task.chunkPos, ex); + } + // Failed to build due to chunk not meeting requirement. + IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null + if (casChunk == null || latestChunk.isStillValid()) // That means CAS have been successful + taskToBuild.addLast(task); // Then add back the same old task. + else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. + task.future.complete(null); + count--; + } + long time2 = System.nanoTime(); + if (!allDone) { + //LOGGER.info("Completed {} tasks in {} in this tick", count, Duration.ofNanos(time2 - time)); + } else if (count > 0) { + //LOGGER.info("Completed all {} tasks in {}", count, Duration.ofNanos(time2 - time)); + } + } } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/transform/DataRenderTransformer.java b/core/src/main/java/com/seibel/lod/core/datatype/transform/DataRenderTransformer.java index d37200813..c2ee97931 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/transform/DataRenderTransformer.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/transform/DataRenderTransformer.java @@ -13,7 +13,7 @@ import java.util.concurrent.ExecutorService; //TODO: Merge this with FullToColumnTransformer public class DataRenderTransformer { public static final ExecutorService TRANSFORMER_THREADS - = LodUtil.makeThreadPool(2, "Data/Render Transformer"); + = LodUtil.makeThreadPool(4, "Data/Render Transformer"); public static CompletableFuture transformDataSource(LodDataSource data, IClientLevel level) { return CompletableFuture.supplyAsync(() -> transform(data, level), TRANSFORMER_THREADS); diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java index f68db7c4e..12f3cc0e4 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/DataFileHandler.java @@ -5,6 +5,7 @@ import com.seibel.lod.core.datatype.LodDataSource; import com.seibel.lod.core.datatype.full.ChunkSizedData; import com.seibel.lod.core.datatype.full.FullDataSource; import com.seibel.lod.core.datatype.full.SparseDataSource; +import com.seibel.lod.core.file.MetaFile; import com.seibel.lod.core.level.ILevel; import com.seibel.lod.core.pos.DhLodPos; import com.seibel.lod.core.pos.DhSectionPos; @@ -300,8 +301,10 @@ public class DataFileHandler implements IDataSourceProvider { } @Override - public LodDataSource onDataFileLoaded(LodDataSource source, Function updater, Consumer onUpdated) { + public LodDataSource onDataFileLoaded(LodDataSource source, MetaFile.MetaData metaData, + Consumer onUpdated, Function updater) { boolean changed = updater.apply(source); + if (changed) metaData.dataVersion.incrementAndGet(); if (source instanceof SparseDataSource) { LodDataSource newSource = ((SparseDataSource) source).trySelfPromote(); changed |= newSource != source; diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java index 599f47566..f036864aa 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/DataMetaFile.java @@ -151,7 +151,7 @@ public class DataMetaFile extends MetaFile metaData = makeMetaData(data); return data; }) - .thenApply((data) -> handler.onDataFileLoaded(data, this::applyWriteQueue, this::saveChanges)) + .thenApply((data) -> handler.onDataFileLoaded(data, metaData, this::saveChanges, this::applyWriteQueue)) .whenComplete((v, e) -> { if (e != null) { LOGGER.error("Uncaught error on creation {}: ", path, e); @@ -177,7 +177,7 @@ public class DataMetaFile extends MetaFile // Apply the write queue LodUtil.assertTrue(!inCacheWriteAccessAsserter.get(),"No one should be writing to the cache while we are in the process of " + "loading one into the cache! Is this a deadlock?"); - data = handler.onDataFileLoaded(data, this::applyWriteQueue, this::saveChanges); + data = handler.onDataFileLoaded(data, metaData, this::saveChanges, this::applyWriteQueue); // Finally, return the data. return data; }, handler.getIOExecutor()) diff --git a/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java b/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java index cd1c3ab79..841fcc084 100644 --- a/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java +++ b/core/src/main/java/com/seibel/lod/core/file/datafile/IDataSourceProvider.java @@ -2,6 +2,7 @@ package com.seibel.lod.core.file.datafile; import com.seibel.lod.core.datatype.LodDataSource; import com.seibel.lod.core.datatype.full.ChunkSizedData; +import com.seibel.lod.core.file.MetaFile; import com.seibel.lod.core.pos.DhSectionPos; import java.io.File; @@ -21,7 +22,7 @@ public interface IDataSourceProvider extends AutoCloseable { long getLatestCacheVersion(DhSectionPos sectionPos); CompletableFuture onCreateDataFile(DataMetaFile file); - LodDataSource onDataFileLoaded(LodDataSource source, Function updater, Consumer onUpdated); + LodDataSource onDataFileLoaded(LodDataSource source, MetaFile.MetaData metaData, Consumer onUpdated, Function updater); CompletableFuture onDataFileRefresh(LodDataSource source, Function updater, Consumer onUpdated); File computeDataFilePath(DhSectionPos pos); Executor getIOExecutor(); diff --git a/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/chunk/IChunkWrapper.java b/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/chunk/IChunkWrapper.java index 68a75d1ec..712880311 100644 --- a/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/chunk/IChunkWrapper.java +++ b/core/src/main/java/com/seibel/lod/core/wrapperInterfaces/chunk/IChunkWrapper.java @@ -92,4 +92,6 @@ public interface IChunkWrapper extends IBindable IBiomeWrapper getBiome(int x, int y, int z); DhChunkPos getChunkPos(); + + boolean isStillValid(); }