diff --git a/common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/BatchGenerationEnvironment.java b/common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/BatchGenerationEnvironment.java index 5014c9a52..50331216c 100644 --- a/common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/BatchGenerationEnvironment.java +++ b/common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/BatchGenerationEnvironment.java @@ -22,6 +22,7 @@ package com.seibel.lod.common.wrappers.worldGeneration; import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.lod.common.wrappers.world.ServerLevelWrapper; +import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.*; import com.seibel.lod.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.lod.core.level.IDhServerLevel; import com.seibel.lod.core.config.Config; @@ -36,6 +37,7 @@ import com.seibel.lod.core.util.objects.LodThreadFactory; import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.lod.core.wrapperInterfaces.worldGeneration.AbstractBatchGenerationEnvironmentWrapper; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -44,14 +46,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import com.seibel.lod.common.wrappers.DependencySetupDoneCheck; import com.seibel.lod.common.wrappers.chunk.ChunkWrapper; -import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.ChunkLoader; -import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.LightGetterAdaptor; -import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.LightedWorldGenRegion; -import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.WorldGenLevelLightEngine; import com.seibel.lod.common.wrappers.worldGeneration.step.StepBiomes; import com.seibel.lod.common.wrappers.worldGeneration.step.StepFeatures; import com.seibel.lod.common.wrappers.worldGeneration.step.StepLight; @@ -68,6 +67,8 @@ import net.minecraft.world.level.chunk.ChunkStatus; import net.minecraft.world.level.chunk.LevelChunk; import net.minecraft.world.level.chunk.ProtoChunk; import net.minecraft.world.level.chunk.UpgradeData; +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileStorage; import net.minecraft.world.level.levelgen.DebugLevelSource; import net.minecraft.world.level.levelgen.FlatLevelSource; import net.minecraft.world.level.levelgen.NoiseBasedChunkGenerator; @@ -76,6 +77,8 @@ import net.minecraft.nbt.CompoundTag; import net.minecraft.world.level.lighting.LevelLightEngine; import org.apache.logging.log4j.LogManager; +import javax.annotation.Nullable; + /* Total: 3.135214124s ===================================== @@ -176,6 +179,19 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv public static final int RANGE_TO_RANGE_EMPTY_EXTENSION = 1; public int unknownExceptionCount = 0; public long lastExceptionTriggerTime = 0; + + private AtomicReference regionFileStorageCache = new AtomicReference<>(); + + public RegionFileStorageExternalCache getOrCreateRegionFileCache(RegionFileStorage storage) { + RegionFileStorageExternalCache cache = regionFileStorageCache.get(); + if (cache == null) { + cache = new RegionFileStorageExternalCache(storage); + if (!regionFileStorageCache.compareAndSet(null, cache)) { + cache = regionFileStorageCache.get(); + } + } + return cache; + } public static final LodThreadFactory threadFactory = new LodThreadFactory("DH-Gen-Worker-Thread", Thread.MIN_PRIORITY); @@ -299,9 +315,11 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv Config.Client.WorldGenerator.enableDistantGeneration.set(false); } } - - public static ChunkAccess loadOrMakeChunk(ChunkPos chunkPos, ServerLevel level, LevelLightEngine lightEngine) + + public ChunkAccess loadOrMakeChunk(ChunkPos chunkPos, WorldGenLevelLightEngine lightEngine) { + ServerLevel level = params.level; + CompoundTag chunkData = null; try { @@ -310,7 +328,10 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv #else // Warning: if multiple threads attempt to access this method at the same time, // it can throw EOFExceptions that are caught and logged by Minecraft - chunkData = level.getChunkSource().chunkMap.readChunk(chunkPos); + //chunkData = level.getChunkSource().chunkMap.readChunk(chunkPos); + RegionFileStorage storage = params.level.getChunkSource().chunkMap.worker.storage; + RegionFileStorageExternalCache cache = getOrCreateRegionFileCache(storage); + chunkData = cache.read(chunkPos); #endif } catch (Exception e) @@ -329,6 +350,7 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv { try { + LOAD_LOGGER.info("DistantHorizons: Loading chunk "+chunkPos+" from disk."); return ChunkLoader.read(level, lightEngine, chunkPos, chunkData); } catch (Exception e) @@ -367,7 +389,7 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv ChunkAccess target = null; try { - target = loadOrMakeChunk(chunkPos, params.level, lightEngine); + target = loadOrMakeChunk(chunkPos, lightEngine); } catch (RuntimeException e2) { @@ -607,7 +629,18 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv EVENT_LOGGER.error("Batch Chunk Generator shutdown interrupted! Ignoring child threads...", e); } } - + var r = regionFileStorageCache.get(); + if (r != null) + { + try + { + r.close(); + } + catch (IOException e) + { + EVENT_LOGGER.error("Failed to close region file storage cache!", e); + } + } EVENT_LOGGER.info(BatchGenerationEnvironment.class.getSimpleName()+" shutdown complete."); } diff --git a/common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/mimicObject/RegionFileStorageExternalCache.java b/common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/mimicObject/RegionFileStorageExternalCache.java new file mode 100644 index 000000000..38405c60c --- /dev/null +++ b/common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/mimicObject/RegionFileStorageExternalCache.java @@ -0,0 +1,92 @@ +package com.seibel.lod.common.wrappers.worldGeneration.mimicObject; + +import com.seibel.lod.common.wrappers.worldGeneration.BatchGenerationEnvironment; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.nbt.NbtIo; +import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileStorage; + +import javax.annotation.Nullable; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class RegionFileStorageExternalCache implements AutoCloseable { + public final RegionFileStorage storage; + public static final int MAX_CACHE_SIZE = 16; + + @Override + public void close() throws IOException { + RegionFileCache cache; + while ((cache = regionFileCache.poll()) != null) { + cache.file.close(); + } + } + + static class RegionFileCache { + public final long pos; + public final RegionFile file; + + public RegionFileCache(long pos, RegionFile file) { + this.pos = pos; + this.file = file; + } + } + + public ConcurrentLinkedQueue regionFileCache = new ConcurrentLinkedQueue<>(); + + public RegionFileStorageExternalCache(RegionFileStorage storage) { + this.storage = storage; + } + + @Nullable + public RegionFile getRegionFile(ChunkPos pos) throws IOException { + long posLong = ChunkPos.asLong(pos.getRegionX(), pos.getRegionZ()); + RegionFile rFile; + // First check our custom cache + for (RegionFileCache cache : this.regionFileCache) { + if (cache.pos == posLong) return cache.file; + } + + // Then check vanilla cache + while (true) { + try { + rFile = this.storage.regionCache.getOrDefault(posLong, null); + break; + } catch (ArrayIndexOutOfBoundsException e) { + BatchGenerationEnvironment.LOAD_LOGGER.warn("Concurrency issue detected when getting region file for chunk at " + pos + ". Retrying..."); + } + } + if (rFile != null) return rFile; + + // Otherwise, check if file exist, and if so, add it to the cache + Path p = storage.folder; + if (!Files.exists(p)) return null; + Path rFilePath = p.resolve("r." + pos.getRegionX() + "." + pos.getRegionZ() + ".mca"); + rFile = new RegionFile(rFilePath, p, false); + regionFileCache.add(new RegionFileCache(ChunkPos.asLong(pos.getRegionX(), pos.getRegionZ()), rFile)); + while (regionFileCache.size() > MAX_CACHE_SIZE) { + regionFileCache.poll().file.close(); + } + return rFile; + } + + + @Nullable + public CompoundTag read(ChunkPos pos) throws IOException { + RegionFile file = getRegionFile(pos); + if (file == null) return null; + + try (DataInputStream stream = file.getChunkDataInputStream(pos)) { + if (stream == null) return null; + return NbtIo.read(stream); + } + catch (Throwable e) { + return null; + } + } + +} diff --git a/common/src/main/resources/1_18.lod.accesswidener b/common/src/main/resources/1_18.lod.accesswidener index 205df3528..948446f5a 100644 --- a/common/src/main/resources/1_18.lod.accesswidener +++ b/common/src/main/resources/1_18.lod.accesswidener @@ -33,7 +33,10 @@ accessible method net/minecraft/world/level/lighting/LayerLightEngine queueSecti # lod generation from save file accessible field net/minecraft/server/level/ChunkMap mainThreadExecutor Lnet/minecraft/util/thread/BlockableEventLoop; accessible method net/minecraft/server/level/ChunkMap readChunk (Lnet/minecraft/world/level/ChunkPos;)Lnet/minecraft/nbt/CompoundTag; - +accessible field net/minecraft/world/level/chunk/storage/ChunkStorage worker Lnet/minecraft/world/level/chunk/storage/IOWorker; +accessible field net/minecraft/world/level/chunk/storage/IOWorker storage Lnet/minecraft/world/level/chunk/storage/RegionFileStorage; +accessible field net/minecraft/world/level/chunk/storage/RegionFileStorage regionCache Lit/unimi/dsi/fastutil/longs/Long2ObjectLinkedOpenHashMap; +accessible field net/minecraft/world/level/chunk/storage/RegionFileStorage folder Ljava/nio/file/Path; # grabbing textures accessible field net/minecraft/client/renderer/texture/TextureAtlasSprite animatedTexture Lnet/minecraft/client/renderer/texture/TextureAtlasSprite$AnimatedTexture;