diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 1ea0be2de..a97f1a41c 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -255,26 +255,6 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu @Override public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.get(relativeX, relativeZ); } - @Override - public ArrayList getUngeneratedPosList() - { - ArrayList posList = new ArrayList<>(); - - for (int x = 0; x < this.width; x++) - { - for (int z = 0; z < this.width; z++) - { - SingleColumnFullDataAccessor column = this.get(x,z); - if (column == null || !column.doesColumnExist()) - { - posList.add(new DhSectionPos(this.sectionPos.sectionDetailLevel, this.sectionPos.sectionX + x, this.sectionPos.sectionZ + z)); - } - } - } - - return posList; - } - @Override public void update(ChunkSizedFullDataAccessor chunkDataView) { diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java index ae594e2d4..c706a9dc2 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java @@ -392,26 +392,6 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo return chunk.get(relativeX % this.dataPointsPerSection, relativeZ % this.dataPointsPerSection); } - @Override - public ArrayList getUngeneratedPosList() - { - ArrayList posList = new ArrayList<>(); - - for (int x = 0; x < SECTION_SIZE; x++) - { - for (int z = 0; z < SECTION_SIZE; z++) - { - SingleColumnFullDataAccessor column = this.tryGet(x,z); - if (column == null || !column.doesColumnExist()) - { - posList.add(new DhSectionPos(this.sectionPos.sectionDetailLevel, this.sectionPos.sectionX + x, this.sectionPos.sectionZ + z)); - } - } - } - - return posList; - } - //=========// diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java index e8d22eaa1..832ab2866 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java @@ -265,26 +265,6 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp @Override public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.isColumnNotEmpty.get(relativeX * WIDTH + relativeZ) ? this.get(relativeX, relativeZ) : null; } - @Override - public ArrayList getUngeneratedPosList() - { - ArrayList posList = new ArrayList<>(); - - for (int x = 0; x < this.width; x++) - { - for (int z = 0; z < this.width; z++) - { - SingleColumnFullDataAccessor column = this.get(x,z); - if (column == null || !column.doesColumnExist()) - { - posList.add(new DhSectionPos(this.sectionPos.sectionDetailLevel, this.sectionPos.sectionX + x, this.sectionPos.sectionZ + z)); - } - } - } - - return posList; - } - //=====================// diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java index 567f4d555..971512f9c 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java @@ -5,15 +5,18 @@ import com.seibel.lod.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.lod.core.dataObjects.fullData.accessor.IFullDataAccessor; import com.seibel.lod.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; +import com.seibel.lod.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.lod.core.dataObjects.render.ColumnRenderSource; import com.seibel.lod.core.file.fullDatafile.FullDataMetaFile; import com.seibel.lod.core.level.IDhLevel; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.lod.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.lod.coreapi.util.BitShiftUtil; import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedList; /** * Base for all Full Data Source objects.

@@ -65,10 +68,185 @@ public interface IFullDataSource FullDataPointIdMap getMapping(); - /** @return true if every datapoint in this object has been generated, false otherwise. */ - default boolean isCompletelyGenerated() { return this.getUngeneratedPosList().size() == 0; } - /** @return the list of {@link DhSectionPos} that aren't generated in this data source. */ - ArrayList getUngeneratedPosList(); + /** + * @param highestGeneratorDetailLevel the smallest numerical detail level that the un-generated positions should be split into + * @return the list of {@link DhSectionPos} that aren't generated in this data source. + */ + default ArrayList getUngeneratedPosList(byte highestGeneratorDetailLevel, boolean onlyReturnPositionsTheGeneratorCanAccept) + { + ArrayList posArray = this.getUngeneratedPosList(this.getSectionPos(), highestGeneratorDetailLevel); + + if (onlyReturnPositionsTheGeneratorCanAccept) + { + LinkedList posList = new LinkedList<>(posArray); + + ArrayList cleanedPosArray = new ArrayList<>(); + while (posList.size() > 0) + { + DhSectionPos pos = posList.remove(); + if (pos.sectionDetailLevel > highestGeneratorDetailLevel) + { + pos.forEachChild((childPos) -> { posList.push(childPos); }); + } + else + { + cleanedPosArray.add(pos); + } + } + + return cleanedPosArray; + } + else + { + return posArray; + } + } + default ArrayList getUngeneratedPosList(DhSectionPos quadrantPos, byte highestGeneratorDetailLevel) + { + ArrayList ungeneratedPosList = new ArrayList<>(); + + int sourceRelWidth = this.getWidthInDataPoints(); + + + if (quadrantPos.sectionDetailLevel < highestGeneratorDetailLevel) + { + throw new IllegalArgumentException("detail level lower than world generator can accept."); + } + else if (quadrantPos.sectionDetailLevel == highestGeneratorDetailLevel) + { + // we are at the highest detail level the world generator can accept, + // we either need to generate this whole section, or not at all + + // TODO combine duplicate code + + byte childDetailLevel = (byte) (quadrantPos.sectionDetailLevel); + + int quadrantDetailLevelDiff = this.getSectionPos().sectionDetailLevel - childDetailLevel; + int widthInSecPos = BitShiftUtil.powerOfTwo(quadrantDetailLevelDiff); + int relWidthForSecPos = sourceRelWidth / widthInSecPos; + + DhSectionPos minSecPos = this.getSectionPos().convertToDetailLevel(childDetailLevel); + DhSectionPos inputPos = quadrantPos; + + + + int minRelX = inputPos.sectionX - minSecPos.sectionX; + int minRelZ = inputPos.sectionZ - minSecPos.sectionZ; + int maxRelX = minRelX + 1; + int maxRelZ = minRelZ + 1; + + minRelX = minRelX * relWidthForSecPos; + minRelZ = minRelZ * relWidthForSecPos; + maxRelX = maxRelX * relWidthForSecPos; + maxRelZ = maxRelZ * relWidthForSecPos; + + if (this.getClass() != CompleteFullDataSource.class) + { + int breakpoint= 0; + } + + + boolean quadrantFullyGenerated = true; + for (int relX = minRelX; relX < maxRelX; relX++) + { + for (int relZ = minRelZ; relZ < maxRelZ; relZ++) + { + SingleColumnFullDataAccessor column = this.tryGet(relX, relZ); + if (column == null) + { + int breakpoi= 0; + } + + if (column == null || !column.doesColumnExist())// || column.hasNullDataPoints()) + { + // no data for this relative position + quadrantFullyGenerated = false; + break; + } + } + } + + if (!quadrantFullyGenerated) + { + // at least 1 data point is missing, + // this whole section must be regenerated + ungeneratedPosList.add(quadrantPos); + } + } + else + { + // TODO comment + // TODO combine duplicate code + + byte childDetailLevel = (byte) (quadrantPos.sectionDetailLevel-1); + + for (int i = 0; i < 4; i++) + { + int quadrantDetailLevelDiff = this.getSectionPos().sectionDetailLevel - childDetailLevel; + int widthInSecPos = BitShiftUtil.powerOfTwo(quadrantDetailLevelDiff); + int relWidthForSecPos = sourceRelWidth / widthInSecPos; + + DhSectionPos minSecPos = this.getSectionPos().convertToDetailLevel(childDetailLevel); + DhSectionPos inputPos = quadrantPos.getChildByIndex(i); + + + + int minRelX = inputPos.sectionX - minSecPos.sectionX; + int minRelZ = inputPos.sectionZ - minSecPos.sectionZ; + int maxRelX = minRelX + 1; + int maxRelZ = minRelZ + 1; + + minRelX = minRelX * relWidthForSecPos; + minRelZ = minRelZ * relWidthForSecPos; + maxRelX = maxRelX * relWidthForSecPos; + maxRelZ = maxRelZ * relWidthForSecPos; + + + + boolean quadrantFullyGenerated = true; + boolean quadrantEmpty = true; + for (int relX = minRelX; relX < maxRelX; relX++) + { + for (int relZ = minRelZ; relZ < maxRelZ; relZ++) + { + SingleColumnFullDataAccessor column = this.tryGet(relX, relZ); + if (column == null || !column.doesColumnExist()) // || column.hasNullDataPoints()) + { + // no data for this relative position + quadrantFullyGenerated = false; + } + else + { + // data exists for this pos + quadrantEmpty = false; + } + } + } + + + if (quadrantFullyGenerated) + { + // no generation necessary + continue; + } + else if (quadrantEmpty) + { + // nothing exists for this sub quadrant, add this sub-quadrant's position + ungeneratedPosList.add(inputPos); + } + else + { + // some data exists in this quadrant, but not all that we need + // recurse down to determine which sub-quadrant positions will need generation + + ungeneratedPosList.addAll(this.getUngeneratedPosList(inputPos, highestGeneratorDetailLevel)); + } + + } + } + + return ungeneratedPosList; + } diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java index b4f7e470e..a44055ef6 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataFileHandler.java @@ -30,14 +30,14 @@ import java.util.function.Function; public class FullDataFileHandler implements IFullDataSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - + // TODO add config option to set pool size - final ExecutorService fileHandlerThread = ThreadUtil.makeThreadPool(4, FullDataFileHandler.class.getSimpleName()+"Thread"); - final ConcurrentHashMap files = new ConcurrentHashMap<>(); - final IDhLevel level; - final File saveDir; - AtomicInteger topDetailLevel = new AtomicInteger(-1); - final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; + protected final ExecutorService fileHandlerThread = ThreadUtil.makeThreadPool(4, FullDataFileHandler.class.getSimpleName()+"Thread"); + protected final ConcurrentHashMap fileBySectionPos = new ConcurrentHashMap<>(); + protected final IDhLevel level; + protected final File saveDir; + protected final AtomicInteger topDetailLevel = new AtomicInteger(-1); + protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; @@ -130,13 +130,13 @@ public class FullDataFileHandler implements IFullDataSourceProvider } // Add file to the list of files. this.topDetailLevel.updateAndGet(v -> Math.max(v, fileToUse.pos.sectionDetailLevel)); - this.files.put(pos, fileToUse); + this.fileBySectionPos.put(pos, fileToUse); } } protected FullDataMetaFile getOrMakeFile(DhSectionPos pos) { - FullDataMetaFile metaFile = this.files.get(pos); + FullDataMetaFile metaFile = this.fileBySectionPos.get(pos); if (metaFile == null) { FullDataMetaFile newMetaFile; @@ -149,7 +149,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider LOGGER.error("IOException on creating new data file at {}", pos, e); return null; } - metaFile = this.files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. + metaFile = this.fileBySectionPos.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. if (metaFile == null) { metaFile = newMetaFile; @@ -190,7 +190,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider continue; } - if (this.files.containsKey(subPos)) + if (this.fileBySectionPos.containsKey(subPos)) { allEmpty = false; break outerLoop; @@ -219,7 +219,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider DhSectionPos childPos = pos.getChildByIndex(childIndex); if (CompleteFullDataSource.firstDataPosCanAffectSecond(basePos, childPos)) { - FullDataMetaFile metaFile = this.files.get(childPos); + FullDataMetaFile metaFile = this.fileBySectionPos.get(childPos); if (metaFile != null) { // we have reached a populated leaf node in the quad tree @@ -285,7 +285,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider } private void writeChunkDataToMetaFile(DhSectionPos sectionPos, ChunkSizedFullDataAccessor chunkData) { - FullDataMetaFile metaFile = this.files.get(sectionPos); + FullDataMetaFile metaFile = this.fileBySectionPos.get(sectionPos); if (metaFile != null) { // there is a file for this position @@ -304,7 +304,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider public CompletableFuture flushAndSave() { ArrayList> futures = new ArrayList<>(); - for (FullDataMetaFile metaFile : this.files.values()) + for (FullDataMetaFile metaFile : this.fileBySectionPos.values()) { futures.add(metaFile.flushAndSaveAsync()); } @@ -392,7 +392,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider FileUtil.renameCorruptedFile(metaFile.file); // remove the FullDataMetaFile since the old one was corrupted - this.files.remove(pos); + this.fileBySectionPos.remove(pos); // create a new FullDataMetaFile to write new data to return this.getOrMakeFile(pos); } diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java index 438792977..e1cb72d09 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java @@ -344,7 +344,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile { if (exception != null) { - LOGGER.error("Error refreshing data "+this.pos+": "+exception+" "+exception.getMessage()); + LOGGER.error("Error refreshing data "+this.pos+": "+exception+" "+exception.getMessage(), exception); future.complete(null); this.cachedFullDataSource = new SoftReference<>(null); } diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 7491f2034..7d24659d3 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -9,6 +9,7 @@ import com.seibel.lod.core.generation.tasks.IWorldGenTaskTracker; import com.seibel.lod.core.generation.WorldGenerationQueue; import com.seibel.lod.core.generation.tasks.WorldGenResult; import com.seibel.lod.core.level.IDhServerLevel; +import com.seibel.lod.core.pos.DhLodPos; import com.seibel.lod.core.pos.DhSectionPos; import com.seibel.lod.core.logging.DhLoggerBuilder; import com.seibel.lod.core.util.LodUtil; @@ -24,21 +25,19 @@ import java.util.function.Consumer; public class GeneratedFullDataFileHandler extends FullDataFileHandler { - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); + private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); private final ArrayList onWorldGenTaskCompleteListeners = new ArrayList<>(); - /** + /** * Keeps track of which partially generated {@link IFullDataSource} {@link DhSectionPos}' are waiting to be generated. * This is done to prevent sending duplicate generation requests for the same position. */ private final HashSet incompleteSourceGenRequests = new HashSet<>(); - - - public GeneratedFullDataFileHandler(IDhServerLevel level, File saveRootDir) { super(level, saveRootDir); } + public GeneratedFullDataFileHandler(IDhServerLevel level, File saveRootDir) { super(level, saveRootDir); } @@ -46,11 +45,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // data // //======// + @Override public CompletableFuture read(DhSectionPos pos) { - return super.read(pos).whenComplete((fullDataSource, ex) -> + return super.read(pos).whenComplete((fullDataSource, ex) -> { - this.checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource); + this.DISABLED_checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource); }); } @@ -61,11 +61,11 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler //==================// /** Assumes there isn't a pre-existing queue. */ - public void setGenerationQueue(WorldGenerationQueue newWorldGenQueue) + public void setGenerationQueue(WorldGenerationQueue newWorldGenQueue) { - boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); - LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); - } + boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); + LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); + } public void clearGenerationQueue() { this.worldGenQueueRef.set(null); } @@ -76,6 +76,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler //=================// public void addWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.add(listener); } + public void removeWorldGenCompleteListener(IOnWorldGenCompleteListener listener) { this.onWorldGenTaskCompleteListeners.remove(listener); } @@ -84,17 +85,17 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // events // //========// - @Override - public CompletableFuture onCreateDataFile(FullDataMetaFile file) + @Override + public CompletableFuture onCreateDataFile(FullDataMetaFile file) { - DhSectionPos pos = file.pos; - + DhSectionPos pos = file.pos; + ArrayList existingFiles = new ArrayList<>(); - ArrayList missingPositions = new ArrayList<>(); + ArrayList missingPositions = new ArrayList<>(); this.getDataFilesForPosition(pos, pos, existingFiles, missingPositions); // confirm the quad tree has at least one node in it - LodUtil.assertTrue(!missingPositions.isEmpty() || !existingFiles.isEmpty()); + LodUtil.assertTrue(!missingPositions.isEmpty() || !existingFiles.isEmpty()); // determine the type of dataSource that should be used for this position @@ -109,76 +110,84 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } - if (missingPositions.size() == 1 && existingFiles.isEmpty() && missingPositions.get(0).equals(pos)) + if (missingPositions.size() == 1 && existingFiles.isEmpty() && missingPositions.get(0).equals(pos)) { - // No LOD data exists for this position yet + // No LOD data exists for this position yet - WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null) + WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue != null) { + this.incompleteSourceGenRequests.add(pos); + // queue this section to be generated GenTask genTask = new GenTask(pos, new WeakReference<>(incompleteFullDataSource)); - worldGenQueue.submitGenTask(incompleteFullDataSource.getSectionPos().getSectionBBoxPos(), incompleteFullDataSource.getDataDetailLevel(), genTask) - .whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos)); - } + worldGenQueue.submitGenTask(new DhLodPos(pos), incompleteFullDataSource.getDataDetailLevel(), genTask) + .whenComplete((genTaskResult, ex) -> + { + this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos); + this.fireOnGenPosSuccessListeners(pos); + this.incompleteSourceGenRequests.remove(pos); + }); + } // return the empty dataSource (it will be populated later) - return CompletableFuture.completedFuture(incompleteFullDataSource); - } + return CompletableFuture.completedFuture(incompleteFullDataSource); + } else { // LOD data exists for this position // create the missing metaData files - for (DhSectionPos missingPos : missingPositions) + for (DhSectionPos missingPos : missingPositions) { - FullDataMetaFile newFile = this.getOrMakeFile(missingPos); - if (newFile != null) + FullDataMetaFile newFile = this.getOrMakeFile(missingPos); + if (newFile != null) { existingFiles.add(newFile); } - } + } -// LOGGER.debug("Creating "+pos+" from sampling "+existingFiles.size()+" files: "+existingFiles); + // LOGGER.debug("Creating "+pos+" from sampling "+existingFiles.size()+" files: "+existingFiles); // read in the existing data final ArrayList> loadDataFutures = new ArrayList<>(existingFiles.size()); - for (FullDataMetaFile existingFile : existingFiles) + for (FullDataMetaFile existingFile : existingFiles) { - loadDataFutures.add(existingFile.loadOrGetCachedDataSourceAsync() - .exceptionally((ex) -> /*Ignore file read errors*/null) - .thenAccept((fullDataSource) -> - { - if (fullDataSource == null) + loadDataFutures.add(existingFile.loadOrGetCachedDataSourceAsync() + .exceptionally((ex) -> /*Ignore file read errors*/null) + .thenAccept((fullDataSource) -> { - return; - } - - this.checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource); - - //LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); - incompleteFullDataSource.sampleFrom(fullDataSource); - }) - ); - } + if (fullDataSource == null) + { + return; + } + + this.DISABLED_checkIfSectionNeedsAdditionalGeneration(pos, fullDataSource); + + //LOGGER.info("Merging data from {} into {}", data.getSectionPos(), pos); + incompleteFullDataSource.sampleFrom(fullDataSource); + }) + ); + } - return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0])) - .thenApply((voidValue) -> incompleteFullDataSource.tryPromotingToCompleteDataSource()); - } - } - - /** - * Checks if the given {@link IFullDataSource} is fully generated and - * if it isn't, new world gen request(s) will be created. - */ - private void checkIfSectionNeedsAdditionalGeneration(DhSectionPos pos, IFullDataSource fullDataSource) - { - boolean generateSection = fullDataSource == null || (!fullDataSource.isCompletelyGenerated() && !incompleteSourceGenRequests.contains(pos)); - if (!generateSection) - { - // this section doesn't need to be generated - return; + return CompletableFuture.allOf(loadDataFutures.toArray(new CompletableFuture[0])) + .thenApply((voidValue) -> incompleteFullDataSource.tryPromotingToCompleteDataSource()); } + } + + /** + * Checks if the given {@link IFullDataSource} is fully generated and + * if it isn't, creates the necessary world gen request(s) to finish it.
+ * Should be used to fill out partially generated {@link IFullDataSource}'s, + * not populate empty ones. + */ + private void DISABLED_checkIfSectionNeedsAdditionalGeneration(DhSectionPos pos, IFullDataSource fullDataSource) + { + // FIXME method is disabled since it will often cause duplicate world gen requests which cause other issues + // and James is out of ideas for how to fix it. + if (true) + return; + WorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue == null) @@ -188,34 +197,63 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } - // the data source could be null if no file exists for this position - if (fullDataSource == null) + if (fullDataSource == null || fullDataSource.isEmpty()) { - if (pos.sectionDetailLevel <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL) - { - fullDataSource = HighDetailIncompleteFullDataSource.createEmpty(pos); - } - else - { - fullDataSource = LowDetailIncompleteFullDataSource.createEmpty(pos); - } + // none of the data source has been generated, those generation requests should be handled elsewhere + return; + } + else if (this.incompleteSourceGenRequests.contains(pos)) + { + return; + } + + ArrayList ungeneratedPosList = fullDataSource.getUngeneratedPosList(worldGenQueue.maxGranularity, true); + if (ungeneratedPosList.size() == 0) + { + // this section doesn't need to be generated + return; } - incompleteSourceGenRequests.add(pos); - //LOGGER.info("["+ungeneratedPosList.size()+"] missing sub positions for pos: ["+pos+"]. Number of gen requests queued: ["+queuedGenRequests.size()+"]."); - - // note: this will potentially re-generate terrain, however due to the generator setup this is currently unavoidable and probably not worth worrying about - GenTask genTask = new GenTask(pos, new WeakReference<>(fullDataSource)); - worldGenQueue.submitGenTask(fullDataSource.getSectionPos().getSectionBBoxPos(), fullDataSource.getDataDetailLevel(), genTask) + //LOGGER.info("["+ungeneratedPosList.size()+"] missing sub positions for pos: ["+pos+"]. Number of gen requests queued: ["+this.incompleteSourceGenRequests.size()+"]."); + + List> futureList = new ArrayList<>(); + for (DhSectionPos ungenChildPos : ungeneratedPosList) + { + // don't queue the same section twice + if (this.incompleteSourceGenRequests.contains(ungenChildPos)) + { + continue; + } + this.incompleteSourceGenRequests.add(ungenChildPos); + + + // FIXME this can cause duplicate terrain generation requests + GenTask genTask = new GenTask(ungenChildPos, new WeakReference<>(fullDataSource)); + CompletableFuture future = worldGenQueue.submitGenTask(new DhLodPos(ungenChildPos), fullDataSource.getDataDetailLevel(), genTask) .whenComplete((genTaskResult, ex) -> { - incompleteSourceGenRequests.remove(pos); - //LOGGER.info("Partial generation completed for pos: ["+pos+"]. Remaining gen requests queued: ["+queuedGenRequests.size()+"]."); - - this.onWorldGenTaskComplete(genTaskResult, ex, genTask, pos); + //LOGGER.info("Partial generation completed for pos: ["+pos+"]. Remaining gen requests queued: ["+this.incompleteSourceGenRequests.size()+"]."); + + this.onWorldGenTaskComplete(genTaskResult, ex, genTask, ungenChildPos); + this.fireOnGenPosSuccessListeners(pos); + this.incompleteSourceGenRequests.remove(ungenChildPos); }); + futureList.add(future); + } + + + if (futureList.size() != 0) + { + CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])) + .whenComplete((genTaskResult, ex) -> + { + // parent pos has completed generation + this.fireOnGenPosSuccessListeners(pos); + this.incompleteSourceGenRequests.remove(pos); + }); + } } private void onWorldGenTaskComplete(WorldGenResult genTaskResult, Throwable exception, GenTask genTask, DhSectionPos pos) @@ -231,33 +269,39 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler else if (genTaskResult.success) { // generation completed, update the files and listener(s) - -// LOGGER.info("gen task completed for pos: ["+pos+"]."); - this.files.get(genTask.pos).flushAndSaveAsync(); - - // fire the event listeners - for (IOnWorldGenCompleteListener listener : this.onWorldGenTaskCompleteListeners) - { - listener.onWorldGenTaskComplete(genTask.pos); - } - -// this.files.get(genTask.pos).metaData.dataVersion.incrementAndGet(); + this.fireOnGenPosSuccessListeners(pos); return; } else { // generation didn't complete - - // if the generation task was split up into smaller positions, wait for them to complete - for (CompletableFuture siblingFuture : genTaskResult.childFutures) - { - siblingFuture.whenComplete((siblingGenTaskResult, siblingEx) -> this.onWorldGenTaskComplete(siblingGenTaskResult, siblingEx, genTask, pos)); - } + } + + + // if the generation task was split up into smaller positions, add the on-complete event to them + for (CompletableFuture siblingFuture : genTaskResult.childFutures) + { + siblingFuture.whenComplete((siblingGenTaskResult, siblingEx) -> this.onWorldGenTaskComplete(siblingGenTaskResult, siblingEx, genTask, pos)); } genTask.releaseStrongReference(); } + private void fireOnGenPosSuccessListeners(DhSectionPos pos) + { + //LOGGER.info("gen task completed for pos: ["+pos+"]."); + + this.fileBySectionPos.get(pos).flushAndSaveAsync() + .whenComplete((voidObj, ex) -> + { + // fire the event listeners + for (IOnWorldGenCompleteListener listener : this.onWorldGenTaskCompleteListeners) + { + listener.onWorldGenTaskComplete(pos); + } + }); + } + //================// @@ -292,11 +336,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler if (this.loadedTargetFullDataSource == null) { this.loadedTargetFullDataSource = this.targetFullDataSourceRef.get(); - if (this.loadedTargetFullDataSource == null) - { - return null; - } } + if (this.loadedTargetFullDataSource == null) + { + return null; + } + return (chunkSizedFullDataSource) -> { @@ -311,8 +356,8 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } - /** - * used by external event listeners
+ /** + * used by external event listeners
* TODO may or may not be best to have this in a separate file */ @FunctionalInterface diff --git a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java index 3073b98b8..99af3c0dd 100644 --- a/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/renderfile/RenderMetaDataFile.java @@ -30,7 +30,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile * When clearing, don't set to null, instead create a SoftReference containing null. * This will make null checks simpler. */ - private SoftReference cachedRenderDataSource = new SoftReference<>(null); + private SoftReference cachedRenderDataSourceRef = new SoftReference<>(null); private final RenderSourceFileHandler fileHandler; private boolean doesFileExist; @@ -110,7 +110,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile private CompletableFuture getCachedDataSourceAsync() { // attempt to get the cached data source - ColumnRenderSource cachedRenderDataSource = this.cachedRenderDataSource.get(); + ColumnRenderSource cachedRenderDataSource = this.cachedRenderDataSourceRef.get(); if (cachedRenderDataSource != null) { return this.fileHandler.onReadRenderSourceLoadedFromCacheAsync(this, cachedRenderDataSource) @@ -127,7 +127,6 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile public CompletableFuture loadOrGetCachedDataSourceAsync(Executor fileReaderThreads, IDhLevel level) { - CompletableFuture getCachedFuture = this.getCachedDataSourceAsync(); if (getCachedFuture != null) { @@ -157,12 +156,12 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile { LOGGER.error("Uncaught error on creation {}: ", this.file, ex); loadRenderSourceFuture.complete(null); - this.cachedRenderDataSource = new SoftReference<>(null); + this.cachedRenderDataSourceRef = new SoftReference<>(null); } else { loadRenderSourceFuture.complete(renderSource); - this.cachedRenderDataSource = new SoftReference<>(renderSource); + this.cachedRenderDataSourceRef = new SoftReference<>(renderSource); } }); } @@ -196,12 +195,12 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile { LOGGER.error("Error loading file {}: ", this.file, ex); loadRenderSourceFuture.complete(null); - this.cachedRenderDataSource = new SoftReference<>(null); + this.cachedRenderDataSourceRef = new SoftReference<>(null); } else { loadRenderSourceFuture.complete(renderSource); - this.cachedRenderDataSource = new SoftReference<>(renderSource); + this.cachedRenderDataSourceRef = new SoftReference<>(renderSource); } }); } diff --git a/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java index 194c015d6..c35f5c540 100644 --- a/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/lod/core/generation/WorldGenerationQueue.java @@ -6,6 +6,7 @@ import com.seibel.lod.core.config.Config; import com.seibel.lod.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.lod.core.dataObjects.transformers.LodDataBuilder; import com.seibel.lod.core.dependencyInjection.SingletonInjector; +import com.seibel.lod.core.file.fullDatafile.GeneratedFullDataFileHandler; import com.seibel.lod.core.generation.tasks.*; import com.seibel.lod.core.pos.*; import com.seibel.lod.core.util.ThreadUtil; @@ -19,6 +20,7 @@ import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import org.apache.logging.log4j.Logger; import java.io.Closeable; +import java.lang.ref.WeakReference; import java.util.*; import java.util.concurrent.*; import java.util.function.Consumer; @@ -35,11 +37,13 @@ public class WorldGenerationQueue implements Closeable private final ConcurrentHashMap inProgressGenTasksByLodPos = new ConcurrentHashMap<>(); // granularity is the detail level for batching world generator requests together - private final byte maxGranularity; - private final byte minGranularity; + public final byte maxGranularity; + public final byte minGranularity; - private final byte maxDataDetail; - private final byte minDataDetail; + /** largest numerical detail level allowed */ + final byte largestDataDetail; + /** lowest numerical detail level allowed */ + public final byte smallestDataDetail; /** If not null this generator is in the process of shutting down */ private volatile CompletableFuture generatorClosingFuture = null; @@ -54,24 +58,26 @@ public class WorldGenerationQueue implements Closeable private DhBlockPos2D generationTargetPos = DhBlockPos2D.ZERO; /** can be used for debugging how many tasks are currently in the queue */ private int numberOfTasksQueued = 0; - /** - * Settings this to true will cause the system to queue the first generation request it can find, - * this improves generation thread feeding efficiency, but can potentially cause generation requests to queue out of order (IE they may not be closest to farthest).

- * - * Setting this to false will cause the system to queue the closest generation request it can find.
- * This will reduce generation queuing efficiency. - */ - private boolean queueFirstGenerationRequestFound = true; + + // debug variables to test for duplicate world generator requests // + /** limits how many of the previous world gen requests we should track */ + private static final int MAX_ALREADY_GENERATED_COUNT = 400; + private final HashSet alreadyGeneratedPosHashSet = new HashSet<>(MAX_ALREADY_GENERATED_COUNT); + private final Queue alreadyGeneratedPosQueue = new LinkedList<>(); + //==============// + // constructors // + //==============// + public WorldGenerationQueue(IDhApiWorldGenerator generator) { this.generator = generator; this.maxGranularity = generator.getMaxGenerationGranularity(); this.minGranularity = generator.getMinGenerationGranularity(); - this.maxDataDetail = generator.getLargestDataDetailLevel(); - this.minDataDetail = generator.getSmallestDataDetailLevel(); + this.largestDataDetail = generator.getLargestDataDetailLevel(); + this.smallestDataDetail = generator.getSmallestDataDetailLevel(); int treeWidth = Config.Client.Graphics.Quality.lodChunkRenderDistance.get() * LodUtil.CHUNK_WIDTH * 2; // TODO the *2 is to allow for generation edge cases, and should probably be removed at some point byte treeMinDetailLevel = LodUtil.BLOCK_DETAIL_LEVEL; // the tree shouldn't need to go this low, but just in case @@ -105,13 +111,13 @@ public class WorldGenerationQueue implements Closeable // make sure the generator can provide the requested position - if (requiredDataDetail < this.minDataDetail) + if (requiredDataDetail < this.smallestDataDetail) { throw new UnsupportedOperationException("Current generator does not meet requiredDataDetail level"); } - if (requiredDataDetail > this.maxDataDetail) + if (requiredDataDetail > this.largestDataDetail) { - requiredDataDetail = this.maxDataDetail; + requiredDataDetail = this.largestDataDetail; } // TODO what does this assert mean? @@ -358,9 +364,36 @@ public class WorldGenerationQueue implements Closeable DhLodPos taskPos = inProgressTaskGroup.group.pos; byte granularity = (byte) (taskPos.detailLevel - taskDetailLevel); LodUtil.assertTrue(granularity >= this.minGranularity && granularity <= this.maxGranularity); - LodUtil.assertTrue(taskDetailLevel >= this.minDataDetail && taskDetailLevel <= this.maxDataDetail); + LodUtil.assertTrue(taskDetailLevel >= this.smallestDataDetail && taskDetailLevel <= this.largestDataDetail); DhChunkPos chunkPosMin = new DhChunkPos(taskPos.getCornerBlockPos()); + + + // Could be enabled (change the if-false to true) to test for duplicate world generator requests + if (false) + { + if (this.alreadyGeneratedPosHashSet.contains(inProgressTaskGroup.group.pos)) + { + // temporary solution to prevent generating the same section multiple times + LOGGER.warn("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping..."); + + // FIXME this prevents sections from generating, or from being updated in the LodQuadTree, James isn't sure which issue it is + inProgressTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateFail())); + return; + } + + this.alreadyGeneratedPosHashSet.add(inProgressTaskGroup.group.pos); + this.alreadyGeneratedPosQueue.add(inProgressTaskGroup.group.pos); + + + // remove extra tracked positions + while (this.alreadyGeneratedPosQueue.size() > MAX_ALREADY_GENERATED_COUNT) + { + DhLodPos posToRemove = this.alreadyGeneratedPosQueue.poll(); + this.alreadyGeneratedPosHashSet.remove(posToRemove); + } + } + //LOGGER.info("Generating section "+taskPos+" with granularity "+granularity+" at "+chunkPosMin); this.numberOfTasksQueued++; @@ -371,7 +404,8 @@ public class WorldGenerationQueue implements Closeable if (exception != null) { // don't log the shutdown exceptions - if (!UncheckedInterruptedException.isThrowableInterruption(exception) && !(exception instanceof CancellationException || exception.getCause() instanceof CancellationException)) + if (!UncheckedInterruptedException.isThrowableInterruption(exception) + && !(exception instanceof CancellationException || exception.getCause() instanceof CancellationException)) { LOGGER.error("Error generating data for section "+taskPos, exception); }