Fix critical issue causing chunk to lod build extremely slow, and also partly fix sparse data source loading (where I used wrong array instead of loaded data array.) Also improve ChunkToLodBuilder building loops to support multithreaded building

core/src/main/java/com/seibel/lod/core/datatype/full/SparseDataSource.java

@@ -280,7 +280,7 @@ public class SparseDataSource implements LodDataSource {
             FullArrayView[] objectChunks = new FullArrayView[chunks*chunks];
             for (int i=0; i<dataChunks.length; i++) {
                 if (dataChunks[i] == null) continue;
-                objectChunks[i] = new FullArrayView(mapping, new long[dataPerChunk * dataPerChunk][], dataPerChunk);
+                objectChunks[i] = new FullArrayView(mapping, dataChunks[i], dataPerChunk);
             }
 
             return new SparseDataSource(dataFile.pos, mapping, objectChunks);

core/src/main/java/com/seibel/lod/core/datatype/transform/ChunkToLodBuilder.java

@@ -1,6 +1,8 @@
 package com.seibel.lod.core.datatype.transform;
 
+import java.time.Duration;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import com.seibel.lod.core.datatype.full.ChunkSizedData;
 import com.seibel.lod.core.level.ILevel;
@@ -26,8 +28,8 @@ public class ChunkToLodBuilder {
     }
     private final ConcurrentHashMap<DhChunkPos, IChunkWrapper> latestChunkToBuild = new ConcurrentHashMap<>();
     private final ConcurrentLinkedDeque<Task> taskToBuild = new ConcurrentLinkedDeque<>();
-    private final ExecutorService executor = LodUtil.makeSingleThreadPool(ChunkToLodBuilder.class);
-    private final EventLoop ticker = new EventLoop(executor, this::_tick);
+    private final ExecutorService executor = LodUtil.makeThreadPool(4, ChunkToLodBuilder.class);
+    private final AtomicInteger runningCount = new AtomicInteger(0);
 
     public CompletableFuture<ChunkSizedData> tryGenerateData(IChunkWrapper chunk) {
         if (chunk == null) throw new NullPointerException("ChunkWrapper cannot be null!");
@@ -43,33 +45,48 @@ public class ChunkToLodBuilder {
     }
 
     public void tick() {
-        ticker.tick();
+        while (true) {
+            if (runningCount.get() > 8192) return;
+            Task task = taskToBuild.pollFirst();
+            if (task == null) return; // There's no jobs.
+            IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation
+            if (latestChunk == null) {
+                LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos);
+                task.future.complete(null);
+                return;
+            }
+
+            runningCount.incrementAndGet();
+            CompletableFuture.supplyAsync(() -> {
+                long time = System.nanoTime();
+                if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) {
+                    ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk);
+                    if (data != null) {
+                        long time2 = System.nanoTime();
+                        LOGGER.info("Processed Task at {} using {}", task.chunkPos, Duration.ofNanos(time2 - time));
+                        task.future.complete(data);
+                        return true;
+                    }
+                }
+                return false;
+            }, executor).handle((b, ex) -> {
+                runningCount.decrementAndGet();
+                if (ex == null && b) return true;
+                if (ex != null) {
+                    LOGGER.error("Error while processing Task at {}!", task.chunkPos, ex);
+                }
+                // Failed to build due to chunk not meeting requirement.
+                IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null
+                if (casChunk == null) // That means CAS have been successful
+                    taskToBuild.addLast(task); // Then add back the same old task.
+                else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task.
+                    task.future.complete(null);
+                return false;
+            });
+        }
     }
 
     private void _tick() {
-        Task task = taskToBuild.pollFirst();
-        if (task == null) return; // There's no jobs.
-        IChunkWrapper latestChunk = latestChunkToBuild.remove(task.chunkPos); // Basically an Exchange operation
-        if (latestChunk == null) {
-            LOGGER.error("Somehow Task at {} has latestChunk as null! Skipping task!", task.chunkPos);
-            task.future.complete(null);
-            return;
-        }
-
-        if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) {
-            ChunkSizedData data = LodDataBuilder.createChunkData(latestChunk);
-            if (data != null) {
-                task.future.complete(data);
-                return;
-            }
-        }
-
-        // Failed to build due to chunk not meeting requirement.
-        IChunkWrapper casChunk = latestChunkToBuild.putIfAbsent(task.chunkPos, latestChunk); // CAS operation with expected=null
-        if (casChunk == null) // That means CAS have been successful
-            taskToBuild.addLast(task); // Then add back the same old task.
-        else // Else, it means someone managed to sneak in a new gen request in this pos. Then lets drop this old task.
-            task.future.complete(null);
     }
 
 }

core/src/main/java/com/seibel/lod/core/datatype/transform/LodDataBuilder.java

@@ -58,6 +58,7 @@ public class LodDataBuilder {
 
     public static boolean canGenerateLodFromChunk(IChunkWrapper chunk)
     {
+        //return true;
         return chunk != null &&
                 chunk.isLightCorrect();
     }

core/src/main/java/com/seibel/lod/core/file/renderfile/RenderFileHandler.java

@@ -140,6 +140,10 @@ public class RenderFileHandler implements IRenderSourceProvider {
      */
     @Override
     public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) {
+        if (chunkData.getBBoxLodPos().convertUpwardsTo((byte)6).equals(new DhLodPos((byte)6, 10, -11))) {
+            int doNothing = 0;
+        }
+
         recursive_write(sectionPos,chunkData);
         dataSourceProvider.write(sectionPos, chunkData);
     }

core/src/main/java/com/seibel/lod/core/util/objects/EventLoop.java

@@ -40,6 +40,7 @@ public class EventLoop implements AutoCloseable {
             future.cancel(true);
         }
         future = null;
+        executorService.shutdown();
     }
     public boolean isRunning() {
         return future != null && !future.isDone();