From 007749c9ef07981d956db96272b540daecd799ce Mon Sep 17 00:00:00 2001 From: TomTheFurry Date: Mon, 26 Jun 2023 23:20:02 +0800 Subject: [PATCH] Optimization on load time, fix gl bug, and improve transparency --- .../fullDatafile/FullDataFileHandler.java | 153 ++++++++++----- .../GeneratedFullDataFileHandler.java | 19 +- .../RemoteFullDataFileHandler.java | 3 +- .../renderfile/RenderSourceFileHandler.java | 180 +++++++++++------- .../SubDimensionLevelMatcher.java | 2 +- .../core/level/ClientLevelModule.java | 5 +- .../core/level/DhClientLevel.java | 6 +- .../core/level/DhClientServerLevel.java | 4 +- .../core/level/DhServerLevel.java | 4 +- .../core/level/ServerLevelModule.java | 3 +- .../core/pos/DhSectionPos.java | 17 +- .../core/render/LodRenderSection.java | 1 - .../core/render/RenderBufferHandler.java | 85 +++++---- .../core/render/renderer/LodRenderer.java | 4 +- .../core/util/FileScanUtil.java | 22 ++- .../core/util/objects/SortedArraySet.java | 4 + 16 files changed, 317 insertions(+), 195 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index c5b33de06..b124c5cfd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -8,15 +8,18 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIn import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; +import com.seibel.distanthorizons.core.util.FileScanUtil; import com.seibel.distanthorizons.core.util.FileUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; @@ -26,43 +29,74 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; +import static com.seibel.distanthorizons.core.util.FileScanUtil.LOD_FILE_POSTFIX; +import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX; + public class FullDataFileHandler implements IFullDataSourceProvider { + public static final boolean USE_LAZY_LOADING = true; + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); protected static ExecutorService fileHandlerThreadPool; protected static ConfigChangeListener configListener; - - protected final ConcurrentHashMap fileBySectionPos = new ConcurrentHashMap<>(); + + private final ConcurrentHashMap unloadedFiles = new ConcurrentHashMap<>(); + private final ConcurrentHashMap fileBySectionPos = new ConcurrentHashMap<>(); + public void ForEachFile(Consumer consumer) { + fileBySectionPos.values().forEach(consumer); + } + protected final IDhLevel level; protected final File saveDir; protected final AtomicInteger topDetailLevel = new AtomicInteger(-1); protected final int minDetailLevel = CompleteFullDataSource.SECTION_SIZE_OFFSET; - - - public FullDataFileHandler(IDhLevel level, File saveRootDir) + public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this.level = level; - this.saveDir = saveRootDir; + this.saveDir = saveStructure.getFullDataFolder(level.getLevelWrapper()); if (!this.saveDir.exists() && !this.saveDir.mkdirs()) { LOGGER.warn("Unable to create full data folder, file saving may fail."); } + FileScanUtil.scanFiles(saveStructure, level.getLevelWrapper(), this, null); } - - - - /** - * Caller must ensure that this method is called only once, - * and that the {@link FullDataFileHandler} is not used before this method is called. - */ - @Override - public void addScannedFile(Collection detectedFiles) + + /** + * Caller must ensure that this method is called only once, + * and that the {@link FullDataFileHandler} is not used before this method is called. + */ + @Override + public void addScannedFile(Collection detectedFiles) + { + if (USE_LAZY_LOADING) { + lazyAddScannedFile(detectedFiles); + } + else { + immediateAddScannedFile(detectedFiles); + } + } + + private void lazyAddScannedFile(Collection detectedFiles) { + for (File file : detectedFiles) { + try { + DhSectionPos pos = decodePositionByFile(file); + if (pos != null) { + unloadedFiles.put(pos, file); + this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); + } + } + catch (Exception e) { + LOGGER.error("Failed to read data meta file at " + file + ": ", e); + FileUtil.renameCorruptedFile(file); + } + } + } + + private void immediateAddScannedFile(Collection detectedFiles) { HashMultimap filesByPos = HashMultimap.create(); - LOGGER.info("Detected {} valid files in {}", detectedFiles.size(), this.saveDir); - { // Sort files by pos. for (File file : detectedFiles) { @@ -138,29 +172,52 @@ public class FullDataFileHandler implements IFullDataSourceProvider } } - protected FullDataMetaFile getOrMakeFile(DhSectionPos pos) + protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile) { FullDataMetaFile metaFile = this.fileBySectionPos.get(pos); - if (metaFile == null) - { - FullDataMetaFile newMetaFile; - try - { - newMetaFile = new FullDataMetaFile(this, this.level, pos); - } - catch (IOException e) - { - LOGGER.error("IOException on creating new data file at {}", pos, e); - return null; - } - - metaFile = this.fileBySectionPos.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. - if (metaFile == null) - { - metaFile = newMetaFile; + if (metaFile != null) return metaFile; + + File fileToLoad = unloadedFiles.get(pos); + // File does exist, but not loaded yet. + if (fileToLoad != null) { + synchronized (this) { + // Double check locking for loading file, as loading file means also loading the metadata, which + // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the + // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. + metaFile = this.fileBySectionPos.get(pos); + if (metaFile != null) return metaFile; // someone else loaded it already. + try { + metaFile = new FullDataMetaFile(this, this.level, fileToLoad); + this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); + this.fileBySectionPos.put(pos, metaFile); + return metaFile; + } + catch (IOException e) { + LOGGER.error("Failed to read data meta file at " + fileToLoad + ": ", e); + FileUtil.renameCorruptedFile(fileToLoad); + } + finally { + unloadedFiles.remove(pos); + } } - } - return metaFile; + } + if (!allowCreateFile) return null; + // File does not exist, create it. + // In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS + // to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects. + try + { + metaFile = new FullDataMetaFile(this, this.level, pos); + } + catch (IOException e) + { + LOGGER.error("IOException on creating new data file at {}", pos, e); + return null; + } + // This is a CAS with expected null value. + this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); + FullDataMetaFile metaFileCas = this.fileBySectionPos.putIfAbsent(pos, metaFile); + return metaFileCas == null ? metaFile : metaFileCas; } /** @@ -247,14 +304,16 @@ public class FullDataFileHandler implements IFullDataSourceProvider /** * Returns the {@link IFullDataSource} for the given section position.
* The returned data source may be null.

- * + * + * For now, if result is null, it prob means error has occurred when loading or creating the file object.

+ * * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ @Override public CompletableFuture read(DhSectionPos pos) { this.topDetailLevel.updateAndGet(intVal -> Math.max(intVal, pos.sectionDetailLevel)); - FullDataMetaFile metaFile = this.getOrMakeFile(pos); + FullDataMetaFile metaFile = this.getLoadOrMakeFile(pos, true); if (metaFile == null) { return CompletableFuture.completedFuture(null); @@ -386,7 +445,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider protected void makeFiles(ArrayList posList, ArrayList output) { for (DhSectionPos missingPos : posList) { - FullDataMetaFile newFile = this.getOrMakeFile(missingPos); + FullDataMetaFile newFile = this.getLoadOrMakeFile(missingPos, true); if (newFile != null) { output.add(newFile); @@ -427,7 +486,7 @@ public class FullDataFileHandler implements IFullDataSourceProvider // remove the FullDataMetaFile since the old one was corrupted this.fileBySectionPos.remove(pos); // create a new FullDataMetaFile to write new data to - return this.getOrMakeFile(pos); + return this.getLoadOrMakeFile(pos, true); } @Override @@ -455,8 +514,16 @@ public class FullDataFileHandler implements IFullDataSourceProvider } @Override - public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + ".lod"); } - + public File computeDataFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + LOD_FILE_POSTFIX); } + + @Nullable + public DhSectionPos decodePositionByFile(File file) + { + String fileName = file.getName(); + if (!fileName.endsWith(LOD_FILE_POSTFIX)) return null; + fileName = fileName.substring(0, fileName.length() - 4); + return DhSectionPos.deserialize(fileName); + } //==========================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 59dab4d5f..eb5e699fb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -4,7 +4,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedF import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.file.metaData.BaseMetaData; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.WorldGenerationQueue; import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; @@ -13,8 +13,6 @@ import com.seibel.distanthorizons.core.level.IDhServerLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; import com.seibel.distanthorizons.core.util.LodUtil; import org.apache.logging.log4j.Logger; @@ -26,8 +24,6 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; public class GeneratedFullDataFileHandler extends FullDataFileHandler { @@ -40,7 +36,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // Use to hold onto incomplete data sources that are waiting for generation, so that they don't get GC'd before they are generated private final ConcurrentHashMap incompleteDataSources = new ConcurrentHashMap<>(); - public GeneratedFullDataFileHandler(IDhServerLevel level, File saveRootDir) { super(level, saveRootDir); } + public GeneratedFullDataFileHandler(IDhServerLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } @@ -69,17 +65,14 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); LOGGER.info("Set world gen queue for level {} to start.", this.level); - for (FullDataMetaFile metaFile : this.fileBySectionPos.values()) - { + this.ForEachFile(metaFile -> { IFullDataSource data = metaFile.getCachedDataSourceNowOrNull(); - if (data instanceof CompleteFullDataSource) { - continue; - } + if (data instanceof CompleteFullDataSource) return; metaFile.genQueueChecked = false; // unset it so it can be checked again if (data != null) { metaFile.markNeedUpdate(); } - } + }); flushAndSave(); // Trigger an update to the meta files } @@ -130,7 +123,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler if (targetDataDetailLevel > maxSectDataDetailLevel) { ArrayList existingFiles = new ArrayList<>(); byte sectDetailLevel = (byte) (DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + maxSectDataDetailLevel); - pos.forEachChildAtLevel(sectDetailLevel, p -> existingFiles.add(getOrMakeFile(p))); + pos.forEachChildAtLevel(sectDetailLevel, p -> existingFiles.add(getLoadOrMakeFile(p, true))); return sampleFromFiles(dataSource, existingFiles).thenApply(this::tryPromoteDataSource) .exceptionally((e) -> { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java index 394cf0050..575dd2666 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java @@ -1,11 +1,12 @@ package com.seibel.distanthorizons.core.file.fullDatafile; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import java.io.File; public class RemoteFullDataFileHandler extends FullDataFileHandler { - public RemoteFullDataFileHandler(IDhLevel level, File saveRootDir) { super(level, saveRootDir); } + public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 10478093c..6ed346cca 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -1,8 +1,11 @@ package com.seibel.distanthorizons.core.file.renderfile; import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -11,30 +14,34 @@ import com.seibel.distanthorizons.core.dataObjects.transformers.DataRenderTransf import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.util.FileScanUtil; import com.seibel.distanthorizons.core.util.FileUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.config.Config; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; import java.awt.*; import java.io.File; import java.io.IOException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; +import java.util.List; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POSTFIX; + public class RenderSourceFileHandler implements ILodRenderSourceProvider { - public static final String RENDER_FILE_EXTENSION = ".rlod"; + public static final boolean USE_LAZY_LOADING = true; public static final long RENDER_SOURCE_TYPE_ID = ColumnRenderSource.TYPE_ID; private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private final ExecutorService fileHandlerThreadPool = ThreadUtil.makeSingleThreadPool("Render Source File Handler"); + + private final ConcurrentHashMap unloadedFiles = new ConcurrentHashMap<>(); private final ConcurrentHashMap filesBySectionPos = new ConcurrentHashMap<>(); private final IDhClientLevel level; @@ -42,21 +49,18 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider /** This is the lowest (highest numeric) detail level that this {@link RenderSourceFileHandler} is keeping track of. */ AtomicInteger topDetailLevel = new AtomicInteger(6); private final IFullDataSourceProvider fullDataSourceProvider; - - private final ConcurrentHashMap cacheUpdateLockBySectionPos = new ConcurrentHashMap<>(); - - - - - public RenderSourceFileHandler(IFullDataSourceProvider sourceProvider, IDhClientLevel level, File saveRootDir) + + + public RenderSourceFileHandler(IFullDataSourceProvider sourceProvider, IDhClientLevel level, AbstractSaveStructure saveStructure) { this.fullDataSourceProvider = sourceProvider; this.level = level; - this.saveDir = saveRootDir; + this.saveDir = saveStructure.getRenderCacheFolder(level.getLevelWrapper()); if (!this.saveDir.exists() && !this.saveDir.mkdirs()) { LOGGER.warn("Unable to create render data folder, file saving may fail."); } + FileScanUtil.scanFiles(saveStructure, level.getLevelWrapper(), null, this); } @@ -70,7 +74,33 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider * and that the given files are not used before this method is called. */ @Override - public void addScannedFile(Collection newRenderFiles) + public void addScannedFile(Collection detectedFiles) + { + if (USE_LAZY_LOADING) { + lazyAddScannedFile(detectedFiles); + } + else { + immediateAddScannedFile(detectedFiles); + } + } + + private void lazyAddScannedFile(Collection detectedFiles) { + for (File file : detectedFiles) { + try { + DhSectionPos pos = decodePositionByFile(file); + if (pos != null) { + unloadedFiles.put(pos, file); + this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); + } + } + catch (Exception e) { + LOGGER.error("Failed to read data meta file at " + file + ": ", e); + FileUtil.renameCorruptedFile(file); + } + } + } + + private void immediateAddScannedFile(Collection newRenderFiles) { HashMultimap filesByPos = HashMultimap.create(); @@ -89,8 +119,6 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider } } - - // Warn for multiple files with the same pos, and then select the one with the latest timestamp. for (DhSectionPos pos : filesByPos.keySet()) { @@ -148,19 +176,61 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider { fileToUse = metaFiles.iterator().next(); } - - // Add this file to the list of files. this.filesBySectionPos.put(pos, fileToUse); - // increase the lowest detail level if a new lower detail file is found - if (this.topDetailLevel.get() < pos.sectionDetailLevel) - { - this.topDetailLevel.set(pos.sectionDetailLevel); - } + this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); } } - + + protected RenderMetaDataFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile) + { + RenderMetaDataFile metaFile = this.filesBySectionPos.get(pos); + if (metaFile != null) return metaFile; + + File fileToLoad = unloadedFiles.get(pos); + // File does exist, but not loaded yet. + if (fileToLoad != null) { + synchronized (this) { + // Double check locking for loading file, as loading file means also loading the metadata, which + // while not... Very expensive, is still better to avoid multiple threads doing it, and dumping the + // duplicated work to the trash. Therefore, eating the overhead of 'synchronized' is worth it. + metaFile = this.filesBySectionPos.get(pos); + if (metaFile != null) return metaFile; // someone else loaded it already. + try { + metaFile = RenderMetaDataFile.createFromExistingFile(this, fileToLoad); + this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); + this.filesBySectionPos.put(pos, metaFile); + return metaFile; + } + catch (IOException e) { + LOGGER.error("Failed to read render meta file at " + fileToLoad + ": ", e); + FileUtil.renameCorruptedFile(fileToLoad); + } + finally { + unloadedFiles.remove(pos); + } + } + } + if (!allowCreateFile) return null; + // File does not exist, create it. + // In this case, since 'creating' a file object doesn't actually do anything heavy on IO yet, we use CAS + // to avoid overhead of 'synchronized', and eat the mini-overhead of possibly creating duplicate objects. + try + { + metaFile = RenderMetaDataFile.createNewFileForPos(this, pos); + } + catch (IOException e) + { + LOGGER.error("IOException on creating new data file at {}", pos, e); + return null; + } + this.topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetailLevel)); + // This is a CAS with expected null value. + RenderMetaDataFile metaFileCas = this.filesBySectionPos.putIfAbsent(pos, metaFile); + return metaFileCas == null ? metaFile : metaFileCas; + } + /** This call is concurrent. I.e. it supports multiple threads calling this method at the same time. */ @Override public CompletableFuture readAsync(DhSectionPos pos) @@ -171,45 +241,10 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider return CompletableFuture.completedFuture(null); } - - - RenderMetaDataFile metaFile = this.filesBySectionPos.get(pos); - if (metaFile == null) - { - RenderMetaDataFile newMetaFile; - try - { - File renderMetaFile = this.computeRenderFilePath(pos); - boolean renderFileExists = renderMetaFile.exists(); - - if (renderFileExists) - { - newMetaFile = RenderMetaDataFile.createFromExistingFile(this, renderMetaFile); - } - else - { - newMetaFile = RenderMetaDataFile.createNewFileForPos(this, pos); - } - } - catch (IOException e) - { - LOGGER.error("IOException on creating new render file at "+pos, e); - return null; - } - - - metaFile = this.filesBySectionPos.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value. - if (metaFile == null) - { - metaFile = newMetaFile; - } - - // increase the lowest detail level if a new lower detail file was added - if (this.topDetailLevel.get() < pos.sectionDetailLevel) - { - this.topDetailLevel.set(pos.sectionDetailLevel); - } - } + RenderMetaDataFile metaFile = this.getLoadOrMakeFile(pos, true); + + // On error, (when it returns null,) return an empty render source + if (metaFile == null) return CompletableFuture.completedFuture(ColumnRenderSource.createEmptyRenderSource(pos)); return metaFile.loadOrGetCachedDataSourceAsync(this.fileHandlerThreadPool, this.level).handle( (renderSource, exception) -> @@ -258,7 +293,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider for (int ox = 0; ox < width; ox++) { for (int oz = 0; oz < width; oz++) { DhSectionPos sectPos = new DhSectionPos(sectionDetailLevel, sectPosMin.x + ox, sectPosMin.z + oz); - RenderMetaDataFile metaFile = this.filesBySectionPos.get(sectPos); + RenderMetaDataFile metaFile = this.filesBySectionPos.get(sectPos); // bypass the getLoadOrMakeFile(), as we only want in-cache files. if (metaFile != null) { metaFile.updateChunkIfSourceExists(chunk, this.level); @@ -431,9 +466,12 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider } } } - + else { + renderFiles = new File[0]; + } // clear the cached files this.filesBySectionPos.clear(); + addScannedFile(ImmutableList.copyOf(renderFiles)); } @@ -442,7 +480,15 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider // helper methods // //================// - public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RENDER_FILE_EXTENSION);} - + public File computeRenderFilePath(DhSectionPos pos) { return new File(this.saveDir, pos.serialize() + RENDER_FILE_POSTFIX);} + + @Nullable + public DhSectionPos decodePositionByFile(File file) + { + String fileName = file.getName(); + if (!fileName.endsWith(RENDER_FILE_POSTFIX)) return null; + fileName = fileName.substring(0, fileName.length() - RENDER_FILE_POSTFIX.length()); + return DhSectionPos.deserialize(fileName); + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 313dfc5c9..e99c91f13 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -221,7 +221,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable break; } IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), clientLevelWrapper); - IFullDataSourceProvider fileHandler = new FullDataFileHandler(tempLevel, testLevelFolder); + IFullDataSourceProvider fileHandler = new FullDataFileHandler(tempLevel, tempLevel.getSaveStructure()); CompletableFuture testDataSource = fileHandler.read(new DhSectionPos(playerChunkPos)); IFullDataSource lodDataSource = testDataSource.get(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index ba89d9a9c..882d5485c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -84,8 +84,6 @@ public class ClientLevelModule { if (isBuffersDirty) { clientRenderState.renderer.bufferHandler.MarkAllBuffersDirty(); } - - clientRenderState.renderer.bufferHandler.updateQuadTreeRenderSources(); } @@ -256,13 +254,12 @@ public class ClientLevelModule { AbstractSaveStructure saveStructure) { this.levelWrapper = dhClientLevel.getLevelWrapper(); - this.renderSourceFileHandler = new RenderSourceFileHandler(fullDataSourceProvider, dhClientLevel, saveStructure.getRenderCacheFolder(this.levelWrapper)); + this.renderSourceFileHandler = new RenderSourceFileHandler(fullDataSourceProvider, dhClientLevel, saveStructure); this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistance.get() * LodUtil.CHUNK_WIDTH, MC_CLIENT.getPlayerBlockPos().x, MC_CLIENT.getPlayerBlockPos().z, this.renderSourceFileHandler); RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree); - FileScanUtil.scanFiles(saveStructure, this.levelWrapper, fullDataSourceProvider, this.renderSourceFileHandler); this.renderer = new LodRenderer(renderBufferHandler); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 9e63773a9..7dd641339 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -32,13 +32,9 @@ public class DhClientLevel extends DhLevel implements IDhClientLevel public DhClientLevel(AbstractSaveStructure saveStructure, IClientLevelWrapper clientLevelWrapper) { - if (saveStructure.getFullDataFolder(clientLevelWrapper).mkdirs()) - { - LOGGER.warn("unable to create data folder."); - } this.levelWrapper = clientLevelWrapper; this.saveStructure = saveStructure; - dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure.getFullDataFolder(levelWrapper)); + dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure); clientside = new ClientLevelModule(this); clientside.startRenderer(); LOGGER.info("Started DHLevel for "+this.levelWrapper+" with saves at "+this.saveStructure); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index bc037860f..ec96d5d0c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -31,6 +31,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS public final ServerLevelModule serverside; public final ClientLevelModule clientside; + private final IServerLevelWrapper serverLevelWrapper; public IClientLevelWrapper clientLevelWrapper; public DhClientServerLevel(AbstractSaveStructure saveStructure, IServerLevelWrapper serverLevelWrapper) @@ -39,6 +40,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS { LOGGER.warn("unable to create data folder."); } + this.serverLevelWrapper = serverLevelWrapper; serverside = new ServerLevelModule(this, serverLevelWrapper, saveStructure); clientside = new ClientLevelModule(this); LOGGER.info("Started "+DhClientServerLevel.class.getSimpleName()+" for "+ serverLevelWrapper +" with saves at "+saveStructure); @@ -135,7 +137,7 @@ public class DhClientServerLevel extends DhLevel implements IDhClientLevel, IDhS } @Override - public IServerLevelWrapper getServerLevelWrapper() { return serverside.levelWrapper; } + public IServerLevelWrapper getServerLevelWrapper() { return serverLevelWrapper; } @Override public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index 8434d4c8b..84826eb25 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -18,6 +18,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); public final ServerLevelModule serverside; + private final IServerLevelWrapper serverLevelWrapper; public DhServerLevel(AbstractSaveStructure saveStructure, IServerLevelWrapper serverLevelWrapper) { @@ -25,6 +26,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel { LOGGER.warn("unable to create data folder."); } + this.serverLevelWrapper = serverLevelWrapper; serverside = new ServerLevelModule(this, serverLevelWrapper, saveStructure); LOGGER.info("Started DHLevel for {} with saves at {}", serverLevelWrapper, saveStructure); } @@ -83,7 +85,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel } @Override - public IServerLevelWrapper getServerLevelWrapper() { return serverside.levelWrapper; } + public IServerLevelWrapper getServerLevelWrapper() { return serverLevelWrapper; } @Override public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java index 81e5be59c..350adb7e5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java @@ -69,8 +69,7 @@ public class ServerLevelModule { this.parent = parent; this.levelWrapper = levelWrapper; this.saveStructure = saveStructure; - this.dataFileHandler = new GeneratedFullDataFileHandler(parent, saveStructure.getFullDataFolder(levelWrapper)); - FileScanUtil.scanFiles(saveStructure, this.levelWrapper, this.dataFileHandler, null); + this.dataFileHandler = new GeneratedFullDataFileHandler(parent, saveStructure); this.worldGeneratorEnabledConfig = new AppliedConfigState<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index dae62d9d9..ae9edde62 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -3,6 +3,7 @@ package com.seibel.distanthorizons.core.pos; import com.seibel.distanthorizons.core.enums.ELodDirection; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import org.jetbrains.annotations.Nullable; import java.util.function.Consumer; @@ -73,9 +74,8 @@ public class DhSectionPos this.sectionX = dhLodPos.x; this.sectionZ = dhLodPos.z; } - - - + + /** Returns the center for the highest detail level (0) */ public DhLodPos getCenter() { return this.getCenter((byte) 0); } // TODO why does this use detail level 0 instead of this object's detail level? public DhLodPos getCenter(byte returnDetailLevel) @@ -211,8 +211,15 @@ public class DhSectionPos /** Serialize() is different from toString() as it must NEVER be changed, and should be in a short format */ public String serialize() { return "[" + this.sectionDetailLevel + ',' + this.sectionX + ',' + this.sectionZ + ']'; } - - + + @Nullable + public static DhSectionPos deserialize(String value) { + if (value.charAt(0) != '[' || value.charAt(value.length() - 1) != ']') return null; + String[] split = value.substring(1, value.length() - 1).split(","); + if (split.length != 3) return null; + return new DhSectionPos(Byte.parseByte(split[0]), Integer.parseInt(split[1]), Integer.parseInt(split[2])); + + } @Override public String toString() { return "{" + this.sectionDetailLevel + "*" + this.sectionX + "," + this.sectionZ + "}"; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index db5d2557f..41ec163cc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -361,7 +361,6 @@ public class LodRenderSection implements IDebugRenderable } public void markBufferDirty() { - tellNeighborsUpdated(); lastSwapLocalVersion = -1; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java index 595cb53d7..d21feac49 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java @@ -12,6 +12,8 @@ import org.apache.logging.log4j.Logger; import java.util.Comparator; import java.util.Iterator; +import java.util.ListIterator; +import java.util.concurrent.atomic.AtomicBoolean; /** * This object tells the {@link LodRenderer} what buffers to render @@ -26,9 +28,10 @@ public class RenderBufferHandler // TODO: Make sorting go into the update loop instead of the render loop as it doesn't need to be done every frame private SortedArraySet loadedNearToFarBuffers = null; - - - + + private final AtomicBoolean rebuildAllBuffers = new AtomicBoolean(false); + + public RenderBufferHandler(LodQuadTree lodQuadTree) { this.lodQuadTree = lodQuadTree; } @@ -40,7 +43,7 @@ public class RenderBufferHandler * TODO: This might get locked by update() causing move() call. Is there a way to avoid this? * Maybe dupe the base list and use atomic swap on render? Or is this not worth it? */ - public void buildRenderList(Vec3f lookForwardVector) + public void buildRenderListAndUpdateSections(Vec3f lookForwardVector) { ELodDirection[] axisDirections = new ELodDirection[3]; @@ -95,12 +98,20 @@ public class RenderBufferHandler axisDirections[2] = xDir; } } + + Pos2D cPos = lodQuadTree.getCenterBlockPos().toPos2D(); // Now that we have the axis directions, we can sort the render list Comparator farToNearComparator = (loadedBufferA, loadedBufferB) -> { Pos2D aPos = loadedBufferA.pos.getCenter().getCenterBlockPos().toPos2D(); Pos2D bPos = loadedBufferB.pos.getCenter().getCenterBlockPos().toPos2D(); + if (true) { + int aManhattanDistance = aPos.manhattanDist(cPos); + int bManhattanDistance = bPos.manhattanDist(cPos); + return bManhattanDistance - aManhattanDistance; + } + for (ELodDirection axisDirection : axisDirections) { if (axisDirection.getAxis().isVertical()) @@ -135,7 +146,9 @@ public class RenderBufferHandler // Build the sorted list this.loadedNearToFarBuffers = new SortedArraySet<>((a, b) -> -farToNearComparator.compare(a, b)); // TODO is the comparator named wrong? - + + // Update the sections + boolean rebuildAllBuffers = this.rebuildAllBuffers.getAndSet(false); Iterator> nodeIterator = this.lodQuadTree.nodeIterator(); while (nodeIterator.hasNext()) { @@ -143,15 +156,27 @@ public class RenderBufferHandler DhSectionPos sectionPos = node.sectionPos; LodRenderSection renderSection = node.value; - - if (renderSection != null && renderSection.isRenderingEnabled()) - { - AbstractRenderBuffer buffer = renderSection.activeRenderBufferRef.get(); - if (buffer != null) - { - this.loadedNearToFarBuffers.add(new LoadedRenderBuffer(buffer, sectionPos)); + try { + + if (renderSection != null) { + if (rebuildAllBuffers) { + renderSection.markBufferDirty(); + } + renderSection.tryBuildAndSwapBuffer(); + + if (renderSection.isRenderingEnabled()) { + AbstractRenderBuffer buffer = renderSection.activeRenderBufferRef.get(); + if (buffer != null) { + this.loadedNearToFarBuffers.add(new LoadedRenderBuffer(buffer, sectionPos)); + } + } } } + catch (Exception e) + { + LOGGER.error("Error updating QuadTree render source at "+renderSection.pos+".", e); + renderSection.markBufferDirty(); + } } } @@ -163,41 +188,19 @@ public class RenderBufferHandler public void renderTransparent(LodRenderer renderContext) { //TODO: Directional culling - this.loadedNearToFarBuffers.forEach(loadedBuffer -> loadedBuffer.buffer.renderTransparent(renderContext)); + ListIterator iter = this.loadedNearToFarBuffers.listIterator(this.loadedNearToFarBuffers.size()); + while (iter.hasPrevious()) + { + LoadedRenderBuffer loadedBuffer = iter.previous(); + loadedBuffer.buffer.renderTransparent(renderContext); + } } - private boolean rebuildAllBuffers = false; - public void MarkAllBuffersDirty() { - this.rebuildAllBuffers = true; + rebuildAllBuffers.set(true); } - public void updateQuadTreeRenderSources() - { - boolean rebuildAllBuffers = this.rebuildAllBuffers; - this.rebuildAllBuffers = false; - - Iterator> nodeIterator = this.lodQuadTree.nodeIterator(); - while (nodeIterator.hasNext()) - { - LodRenderSection renderSection = nodeIterator.next().value; - try { - if (renderSection != null) - { - if (rebuildAllBuffers) - { - renderSection.markBufferDirty(); - } - renderSection.tryBuildAndSwapBuffer(); - } - } - catch (Exception e) - { - LOGGER.error("Error updating QuadTree render source at "+renderSection.pos+".", e); - } - } - } public void close() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index 163a132d2..92cf2fb42 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -228,7 +228,7 @@ public class LodRenderer //GL32.glEnable( GL32.GL_POLYGON_OFFSET_FILL ); //GL32.glPolygonOffset( 1f, 1f ); - bufferHandler.buildRenderList(this.getLookVector()); + bufferHandler.buildRenderListAndUpdateSections(this.getLookVector()); //===========// // rendering // @@ -253,7 +253,7 @@ public class LodRenderer if (LodRenderer.transparencyEnabled) { GL32.glEnable(GL32.GL_BLEND); GL32.glBlendFunc(GL32.GL_SRC_ALPHA, GL32.GL_ONE_MINUS_SRC_ALPHA); - GL32.glDepthMask(false); // This so that even on incorrect sorting of transparent blocks, it still mostly looks correct + //GL32.glDepthMask(false); // This so that even on incorrect sorting of transparent blocks, it still mostly looks correct bufferHandler.renderTransparent(this); GL32.glDepthMask(true); // Apparently the depth mask state is stored in the FBO, so glState fails to restore it... } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java index 7097c8285..8484d87c9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FileScanUtil.java @@ -2,14 +2,17 @@ package com.seibel.distanthorizons.core.util; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.renderfile.ILodRenderSourceProvider; +import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -19,6 +22,7 @@ public class FileScanUtil private static final Logger LOGGER = DhLoggerBuilder.getLogger(); public static final int MAX_SCAN_DEPTH = 5; public static final String LOD_FILE_POSTFIX = ".lod"; + public static final String RENDER_FILE_POSTFIX = ".rlod"; public static void scanFiles(AbstractSaveStructure saveStructure, ILevelWrapper levelWrapper, @Nullable IFullDataSourceProvider dataSourceProvider, @@ -28,10 +32,11 @@ public class FileScanUtil { try (Stream pathStream = Files.walk(saveStructure.getFullDataFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) { - dataSourceProvider.addScannedFile(pathStream.filter( - path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile() - ).map(Path::toFile).collect(Collectors.toList()) - ); + List files = pathStream.filter( + path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile() + ).map(Path::toFile).collect(Collectors.toList()); + LOGGER.info("Found "+files.size()+" full data files for "+levelWrapper+" in "+saveStructure); + dataSourceProvider.addScannedFile(files); } catch (Exception e) { @@ -43,10 +48,11 @@ public class FileScanUtil { try (Stream pathStream = Files.walk(saveStructure.getRenderCacheFolder(levelWrapper).toPath(), MAX_SCAN_DEPTH)) { - renderSourceProvider.addScannedFile(pathStream.filter(( - path -> path.toFile().getName().endsWith(LOD_FILE_POSTFIX) && path.toFile().isFile()) - ).map(Path::toFile).collect(Collectors.toList()) - ); + List files = pathStream.filter( + path -> path.toFile().getName().endsWith(RENDER_FILE_POSTFIX) && path.toFile().isFile() + ).map(Path::toFile).collect(Collectors.toList()); + LOGGER.info("Found "+files.size()+" render cache files for "+levelWrapper+" in "+saveStructure); + renderSourceProvider.addScannedFile(files); } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/SortedArraySet.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/SortedArraySet.java index 967a555af..b2217385d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/SortedArraySet.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/SortedArraySet.java @@ -67,6 +67,10 @@ public class SortedArraySet implements SortedSet { return list.listIterator(); } + public ListIterator listIterator(int index) { + return list.listIterator(index); + } + @Override public Object[] toArray() {