From 0dafdc527a920d6bb07957a51eaed49183af75bc Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Thu, 18 Aug 2022 19:46:35 +0800 Subject: [PATCH] Continue fixing gen related issues --- .../core/a7/generation/GenerationQueue.java | 239 ++++++++++++------ .../core/a7/level/DhClientServerLevel.java | 2 +- .../ConcurrentQuadCombinableProviderTree.java | 57 ++--- .../core/config/types/ConfigUIComment.java | 2 +- 4 files changed, 189 insertions(+), 111 deletions(-) diff --git a/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java b/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java index 5ca122435..dac556732 100644 --- a/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java +++ b/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java @@ -30,12 +30,14 @@ public class GenerationQueue implements AutoCloseable { if (generator.isBusy()) return; DhLodPos closest = null; long closestDist = Long.MAX_VALUE; + int smallestDetail = Integer.MAX_VALUE; for (DhLodPos key : taskMap.keySet()) { + if (key.detail > smallestDetail) continue; long dist = key.getCenter().distSquared(targetPos); - if (dist < closestDist) { - closest = key; - closestDist = dist; - } + if (key.detail == smallestDetail && dist >= closestDist) continue; + closest = key; + closestDist = dist; + smallestDetail = key.detail; } if (closest != null) { CompletableFuture future = taskMap.remove(closest); @@ -69,77 +71,39 @@ public class GenerationQueue implements AutoCloseable { if (minGenGranularity < 4 || maxGenGranularity < 4) { throw new IllegalStateException("Generation granularity must be at least 4!"); } - byte minUnitDetail = (byte) (dataDetail + minGenGranularity); byte maxUnitDetail = (byte) (dataDetail + maxGenGranularity); + LodUtil.assertTrue(pos.detail >= minUnitDetail && pos.detail <= maxUnitDetail); + byte genGranularity = (byte) (pos.detail - dataDetail); + DHChunkPos chunkPosMin = new DHChunkPos(pos.getCorner()); + logger.info("Generating section {} with granularity {} at {}", pos, genGranularity, chunkPosMin); + int perCallChunksWidth = 1 << (genGranularity - 4); - byte granularity; - int count; - DHChunkPos chunkPosMin; - if (pos.detail < minUnitDetail) { - granularity = minGenGranularity; - count = 1; - chunkPosMin = new DHChunkPos(pos.convertUpwardsTo(minUnitDetail).getCorner()); - } else if (pos.detail > maxUnitDetail) { - granularity = maxGenGranularity; - count = 1 << (pos.detail - maxUnitDetail); - chunkPosMin = new DHChunkPos(pos.getCorner()); - } else { - granularity = (byte) (pos.detail - dataDetail); - count = 1; - chunkPosMin = new DHChunkPos(pos.getCorner()); - } - assert granularity >= minGenGranularity && granularity <= maxGenGranularity; - assert count > 0; - assert granularity >= 4; // Thanks compiler. Guess having a 'always true' warning means I did it right. - logger.info("Generating section {} of size {} with granularity {} at {}", pos, count, granularity, chunkPosMin); - int perCallChunksWidth = 1 << (granularity - 4); - final byte sectionDetail = (byte) (dataDetail + FullDataSource.SECTION_SIZE_OFFSET); - - ArrayList>> futures = new ArrayList<>(count*count); - for (int dx = 0; dx < count; dx++) { - for (int dz = 0; dz < count; dz++) { // TODO: Unroll this loop to yield when generator is busy. - DHChunkPos subCallChunkPosMin = new DHChunkPos(chunkPosMin.x + dx * perCallChunksWidth, chunkPosMin.z + dz * perCallChunksWidth); - CompletableFuture> dataFuture = generator.generate(subCallChunkPosMin, granularity); - futures.add(dataFuture.handle((data, ex) -> { - if (ex != null) { - if (ex instanceof CompletionException) { - ex = ex.getCause(); - } - if (ex instanceof InterruptedException) return null; // Ignore interrupted exceptions. - if (ex instanceof UncheckedInterruptedException) return null; // Ignore unchecked interrupted exceptions. - logger.error("Error generating data for section {}", pos, ex); - return null; - } - LodUtil.assertTrue(data != null); - if (data.gridSize < (1 << (granularity-4))) { - logger.error( - "Generator at {} returned {} by {} chunks but requested granularity was {}, which expect at least {} by {} chunks! ", - pos, data.gridSize, data.gridSize, granularity, perCallChunksWidth, perCallChunksWidth); - return null; - } - - DhLodPos minSectPos = new DhLodPos((byte)(dataDetail+4), data.getFirst().x, data.getFirst().z).convertUpwardsTo(sectionDetail); - DhLodPos maxSectPos = new DhLodPos((byte)(dataDetail+4), data.getLast().x, data.getLast().z).convertUpwardsTo(sectionDetail); - - int sectionCount = (maxSectPos.x - minSectPos.x) + 1; - LodUtil.assertTrue(sectionCount > 0 && sectionCount == (maxSectPos.z - minSectPos.z) + 1); - - logger.info("Writing {} by {} chunks (at {}) with data detail {} to {} by {} sections (at {})", - data.gridSize, data.gridSize, subCallChunkPosMin, dataDetail, - sectionCount, sectionCount, minSectPos); - return data; - })); - } - } + CompletableFuture> dataFuture = generator.generate(chunkPosMin, genGranularity); inProgress.add( - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenApply((v) -> { - GenerationResult result = new GenerationResult(); - for (CompletableFuture> future : futures) { - Collection data = future.join(); - if (data == null) continue; - result.dataList.addAll(data); + dataFuture.handle((data, ex) -> { + if (ex != null) { + if (ex instanceof CompletionException) { + ex = ex.getCause(); + } + if (ex instanceof InterruptedException) return null; // Ignore interrupted exceptions. + if (ex instanceof UncheckedInterruptedException) return null; // Ignore unchecked interrupted exceptions. + logger.error("Error generating data for section {}", pos, ex); + throw new CompletionException("Generation failed", ex); } + LodUtil.assertTrue(data != null); + if (data.gridSize < (1 << (genGranularity-4))) { + logger.error( + "Generator at {} returned {} by {} chunks but requested granularity was {}, which expect at least {} by {} chunks! ", + pos, data.gridSize, data.gridSize, genGranularity, perCallChunksWidth, perCallChunksWidth); + throw new RuntimeException("Generation failed. Generator returned less data than requested!"); + } + logger.info("Completed generating {} by {} chunks to sections that overlaps {}", + data.gridSize, data.gridSize, pos); + return data; + }).thenApply((list) -> { + GenerationResult result = new GenerationResult(); + result.dataList.addAll(list); return result; }).handle((r, e) -> { if (e!=null) resultFuture.completeExceptionally(e); else resultFuture.complete(r); @@ -148,17 +112,132 @@ public class GenerationQueue implements AutoCloseable { ); } +// private void startFuture(DhLodPos pos, CompletableFuture resultFuture) { +// byte dataDetail = generator.getDataDetail(); +// byte minGenGranularity = generator.getMinGenerationGranularity(); +// byte maxGenGranularity = generator.getMaxGenerationGranularity(); +// if (minGenGranularity < 4 || maxGenGranularity < 4) { +// throw new IllegalStateException("Generation granularity must be at least 4!"); +// } +// +// byte minUnitDetail = (byte) (dataDetail + minGenGranularity); +// byte maxUnitDetail = (byte) (dataDetail + maxGenGranularity); +// +// byte granularity; +// int count; +// DHChunkPos chunkPosMin; +// if (pos.detail < minUnitDetail) { +// granularity = minGenGranularity; +// count = 1; +// chunkPosMin = new DHChunkPos(pos.convertUpwardsTo(minUnitDetail).getCorner()); +// } else if (pos.detail > maxUnitDetail) { +// granularity = maxGenGranularity; +// count = 1 << (pos.detail - maxUnitDetail); +// chunkPosMin = new DHChunkPos(pos.getCorner()); +// } else { +// granularity = (byte) (pos.detail - dataDetail); +// count = 1; +// chunkPosMin = new DHChunkPos(pos.getCorner()); +// } +// assert granularity >= minGenGranularity && granularity <= maxGenGranularity; +// assert count > 0; +// assert granularity >= 4; // Thanks compiler. Guess having a 'always true' warning means I did it right. +// logger.info("Generating section {} of size {} with granularity {} at {}", pos, count, granularity, chunkPosMin); +// int perCallChunksWidth = 1 << (granularity - 4); +// final byte sectionDetail = (byte) (dataDetail + FullDataSource.SECTION_SIZE_OFFSET); +// +// ArrayList>> futures = new ArrayList<>(count*count); +// for (int dx = 0; dx < count; dx++) { +// for (int dz = 0; dz < count; dz++) { // TODO: Unroll this loop to yield when generator is busy. +// DHChunkPos subCallChunkPosMin = new DHChunkPos(chunkPosMin.x + dx * perCallChunksWidth, chunkPosMin.z + dz * perCallChunksWidth); +// CompletableFuture> dataFuture = generator.generate(subCallChunkPosMin, granularity); +// futures.add(dataFuture.handle((data, ex) -> { +// if (ex != null) { +// if (ex instanceof CompletionException) { +// ex = ex.getCause(); +// } +// if (ex instanceof InterruptedException) return null; // Ignore interrupted exceptions. +// if (ex instanceof UncheckedInterruptedException) return null; // Ignore unchecked interrupted exceptions. +// logger.error("Error generating data for section {}", pos, ex); +// return null; +// } +// LodUtil.assertTrue(data != null); +// if (data.gridSize < (1 << (granularity-4))) { +// logger.error( +// "Generator at {} returned {} by {} chunks but requested granularity was {}, which expect at least {} by {} chunks! ", +// pos, data.gridSize, data.gridSize, granularity, perCallChunksWidth, perCallChunksWidth); +// return null; +// } +// +// DhLodPos minSectPos = new DhLodPos((byte)(dataDetail+4), data.getFirst().x, data.getFirst().z).convertUpwardsTo(sectionDetail); +// DhLodPos maxSectPos = new DhLodPos((byte)(dataDetail+4), data.getLast().x, data.getLast().z).convertUpwardsTo(sectionDetail); +// +// int sectionCount = (maxSectPos.x - minSectPos.x) + 1; +// LodUtil.assertTrue(sectionCount > 0 && sectionCount == (maxSectPos.z - minSectPos.z) + 1); +// +// logger.info("Writing {} by {} chunks (at {}) with data detail {} to {} by {} sections (at {})", +// data.gridSize, data.gridSize, subCallChunkPosMin, dataDetail, +// sectionCount, sectionCount, minSectPos); +// return data; +// })); +// } +// } +// inProgress.add( +// CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenApply((v) -> { +// GenerationResult result = new GenerationResult(); +// for (CompletableFuture> future : futures) { +// Collection data = future.join(); +// if (data == null) continue; +// result.dataList.addAll(data); +// } +// return result; +// }).handle((r, e) -> { +// if (e!=null) resultFuture.completeExceptionally(e); else resultFuture.complete(r); +// return null; +// }) +// ); +// } + public CompletableFuture generate(DhSectionPos sectionPos) { - DhLodPos lodPos = sectionPos.getSectionBBoxPos(); - return cqcpTree.createOrUseExisting(lodPos, this::createFuture).thenApply( - (result) -> { - if (result == null || result.dataList.isEmpty()) return FullDataSource.createEmpty(sectionPos); - FullDataSource newSource = FullDataSource.createEmpty(sectionPos); - for (ChunkSizedData data : result.dataList) { - newSource.update(data); - } - return newSource; - }); + byte maxGen = (byte) (generator.getMaxGenerationGranularity() + generator.getDataDetail()); + if (sectionPos.sectionDetail > maxGen) { + int count = 1 << (sectionPos.sectionDetail - maxGen); + DhLodPos minPos = sectionPos.getCorner(maxGen); + ArrayList> futures = new ArrayList<>(count*count); + for (int x = 0; x < count; x++) { + for (int z = 0; z < count; z++) { + DhLodPos subPos = new DhLodPos(maxGen, minPos.x + x, minPos.z + z); + futures.add(cqcpTree.createOrUseExisting(subPos, this::createFuture)); + } + } + // FIXME: Does `allOf` have correct behaviour when one of the futures fails? + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply((v) -> { + FullDataSource newSource = FullDataSource.createEmpty(sectionPos); + for (CompletableFuture future : futures) { + try { + GenerationResult result = future.join(); + for (ChunkSizedData data : result.dataList) { + if (data.getBBoxLodPos().overlaps(sectionPos.getSectionBBoxPos())) newSource.update(data); + } + } catch (Exception e) { + // continue + } + } + return newSource; + }); + } else { + DhLodPos lodPos = sectionPos.getSectionBBoxPos(); + return cqcpTree.createOrUseExisting(lodPos, this::createFuture).thenApply( + (result) -> { + if (result == null || result.dataList.isEmpty()) return FullDataSource.createEmpty(sectionPos); + FullDataSource newSource = FullDataSource.createEmpty(sectionPos); + for (ChunkSizedData data : result.dataList) { + if (data.getBBoxLodPos().overlaps(sectionPos.getSectionBBoxPos())) newSource.update(data); + } + return newSource; + }); + } } @Override diff --git a/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java b/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java index a881d7b2f..738666bfd 100644 --- a/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java +++ b/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java @@ -101,12 +101,12 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel { } tree.close(); tree = null; + generationQueue.removeGenerator(); renderBufferHandler.close(); renderBufferHandler = null; renderFileHandler.flushAndSave(); //Ignore the completion feature so that this action is async renderFileHandler.close(); renderFileHandler = null; - generationQueue.removeGenerator(); try { worldGenerator.close(); } catch (Exception e) { diff --git a/src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java b/src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java index 8f184c922..3c9a85da8 100644 --- a/src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java +++ b/src/main/java/com/seibel/lod/core/a7/util/ConcurrentQuadCombinableProviderTree.java @@ -4,14 +4,12 @@ import com.seibel.lod.core.a7.pos.DhLodPos; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.Atomics; import com.seibel.lod.core.util.LodUtil; -import org.apache.commons.lang3.NotImplementedException; import org.apache.logging.log4j.Logger; import java.lang.ref.WeakReference; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -295,34 +293,35 @@ public class ConcurrentQuadCombinableProviderTree> private RootMap rebaseUpward(int targetLevel) { rootMapGlobalLock.writeLock().lock(); - RootMap map = rootMap.get(); - if (map.topLevel >= targetLevel) { - rootMapGlobalLock.writeLock().unlock(); - return map; - } - // At this point, we have exclusive access to the rootMap. - map.clean(); // Clean the map. (Could actually be done with just readLock.) - RootMap newMap = new RootMap<>(map.topLevel + 1); - if (map.roots.isEmpty()) return newMap; // If there are no roots, just return the new map. - map.roots.forEach((pos, nodeRef) -> { - Node node = nodeRef.get(); - if (node == null) return; // If null, ignore that node. - LodUtil.assertTrue(pos.detail+1 == newMap.topLevel); - LodUtil.assertTrue(node.parent.get() == null); - LodUtil.assertTrue(node.pos.equals(pos)); - DhLodPos newPos = pos.convertUpwardsTo((byte) (pos.detail+1)); + try { + RootMap map = rootMap.get(); + if (map.topLevel >= targetLevel) { + return map; + } + // At this point, we have exclusive access to the rootMap. + map.clean(); // Clean the map. (Could actually be done with just readLock.) + RootMap newMap = new RootMap<>(map.topLevel + 1); + map.roots.forEach((pos, nodeRef) -> { + Node node = nodeRef.get(); + if (node == null) return; // If null, ignore that node. + LodUtil.assertTrue(pos.detail+1 == newMap.topLevel); + LodUtil.assertTrue(node.parent.get() == null); + LodUtil.assertTrue(node.pos.equals(pos)); + DhLodPos newPos = pos.convertUpwardsTo((byte) (pos.detail+1)); - // Create the parent node, or if it already exists, use it to set the child node's parent. - // NOTE: While this section is protected by the rootMapGlobalLock, we still need to use the normal - // CAS methods to setAndGet the parent node, as the parent node may be GC'd concurrently by other threads - // who have just completed the node's future, and caused the GC parent chain up to the new map. - Node newParentNode = newMap.setIfNullAndGet(newPos, new Node(newPos, null)); - node.parent.set(newParentNode); - }); - boolean casWorked = rootMap.compareAndSet(map, newMap); - LodUtil.assertTrue(casWorked); - rootMapGlobalLock.writeLock().unlock(); - return newMap; + // Create the parent node, or if it already exists, use it to set the child node's parent. + // NOTE: While this section is protected by the rootMapGlobalLock, we still need to use the normal + // CAS methods to setAndGet the parent node, as the parent node may be GC'd concurrently by other threads + // who have just completed the node's future, and caused the GC parent chain up to the new map. + Node newParentNode = newMap.setIfNullAndGet(newPos, new Node(newPos, null)); + node.parent.set(newParentNode); + }); + boolean casWorked = rootMap.compareAndSet(map, newMap); + LodUtil.assertTrue(casWorked); + return newMap; + } finally { + rootMapGlobalLock.writeLock().unlock(); + } } public void cleanIfNeeded() { diff --git a/src/main/java/com/seibel/lod/core/config/types/ConfigUIComment.java b/src/main/java/com/seibel/lod/core/config/types/ConfigUIComment.java index 86fc64305..59bcdf18f 100644 --- a/src/main/java/com/seibel/lod/core/config/types/ConfigUIComment.java +++ b/src/main/java/com/seibel/lod/core/config/types/ConfigUIComment.java @@ -7,7 +7,7 @@ package com.seibel.lod.core.config.types; */ public class ConfigUIComment extends AbstractConfigType{ public ConfigUIComment(String value) { - super(ConfigEntryAppearance.ONLY_SHOW, value); + super(ConfigEntryAppearance.ONLY_SHOW, value, null); //TODO: Is the listener: null right? } @Override