From b4ea8854a835da4ab35dfd832364bfa9d34d761e Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Sun, 18 Sep 2022 16:29:59 +0800 Subject: [PATCH] Fix critical issue causing chunk to lod build extremely slow, and also partly fix sparse data source loading (where I used wrong array instead of loaded data array.) Also improve ChunkToLodBuilder building loops to support multithreaded building --- .../core/datatype/full/SparseDataSource.java | 2 +- .../datatype/transform/ChunkToLodBuilder.java | 69 ++++++++++++------- .../datatype/transform/LodDataBuilder.java | 1 + .../file/renderfile/RenderFileHandler.java | 4 ++ .../lod/core/util/objects/EventLoop.java | 1 + 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java b/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java index 385c62d8a..54cedfade 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java @@ -280,7 +280,7 @@ public class SparseDataSource implements LodDataSource { FullArrayView[] objectChunks = new FullArrayView[chunks*chunks]; for (int i=0; i latestChunkToBuild = new ConcurrentHashMap<>(); private final ConcurrentLinkedDeque taskToBuild = new ConcurrentLinkedDeque<>(); - private final ExecutorService executor = LodUtil.makeSingleThreadPool(ChunkToLodBuilder.class); - private final EventLoop ticker = new EventLoop(executor, this::_tick); + private final ExecutorService executor = LodUtil.makeThreadPool(4, ChunkToLodBuilder.class); + private final AtomicInteger runningCount = new AtomicInteger(0); public CompletableFuture tryGenerateData(IChunkWrapper chunk) { if (chunk == null) throw new NullPointerException("ChunkWrapper cannot be null!"); @@ -43,33 +45,48 @@ public class ChunkToLodBuilder { } public void tick() { - ticker.tick(); + while (true) { + if (runningCount.get() > 8192) return; + Task task = taskToBuild.pollFirst(); + if (task == null) return; // There's no jobs. + IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation + if (latestChunk == null) { + LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos); + task.future.complete(null); + return; + } + + runningCount.incrementAndGet(); + CompletableFuture.supplyAsync(() -> { + long time = System.nanoTime(); + if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { + ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk); + if (data != null) { + long time2 = System.nanoTime(); + LOGGER.info("Processed Task at {} using {}", task.chunkPos, Duration.ofNanos(time2 - time)); + task.future.complete(data); + return true; + } + } + return false; + }, executor).handle((b, ex) -> { + runningCount.decrementAndGet(); + if (ex == null && b) return true; + if (ex != null) { + LOGGER.error("Error while processing Task at {}!", task.chunkPos, ex); + } + // Failed to build due to chunk not meeting requirement. + IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null + if (casChunk == null) // That means CAS have been successful + taskToBuild.addLast(task); // Then add back the same old task. + else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. + task.future.complete(null); + return false; + }); + } } private void _tick() { - Task task = taskToBuild.pollFirst(); - if (task == null) return; // There's no jobs. - IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation - if (latestChunk == null) { - LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos); - task.future.complete(null); - return; - } - - if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { - ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk); - if (data != null) { - task.future.complete(data); - return; - } - } - - // Failed to build due to chunk not meeting requirement. - IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null - if (casChunk == null) // That means CAS have been successful - taskToBuild.addLast(task); // Then add back the same old task. - else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task. - task.future.complete(null); } } diff --git a/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java b/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java index 67acc4d35..e94d1908a 100644 --- a/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java @@ -58,6 +58,7 @@ public class LodDataBuilder { public static boolean canGenerateLodFromChunk(IChunkWrapper chunk) { + //return true; return chunk != null && chunk.isLightCorrect(); } diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java index f49d06700..258870079 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java @@ -140,6 +140,10 @@ public class RenderFileHandler implements IRenderSourceProvider { */ @Override public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) { + if (chunkData.getBBoxLodPos().convertUpwardsTo((byte)6).equals(new DhLodPos((byte)6, 10, -11))) { + int doNothing = 0; + } + recursive_write(sectionPos,chunkData); dataSourceProvider.write(sectionPos, chunkData); } diff --git a/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java b/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java index b0e63dc2c..0d202656b 100644 --- a/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java +++ b/core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java @@ -40,6 +40,7 @@ public class EventLoop implements AutoCloseable { future.cancel(true); } future = null; + executorService.shutdown(); } public boolean isRunning() { return future != null && !future.isDone();