Merge remote-tracking branch 'origin/main'

common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/BatchGenerationEnvironment.java

@@ -22,6 +22,7 @@ package com.seibel.lod.common.wrappers.worldGeneration;
 
 import com.seibel.lod.api.enums.worldGeneration.EDhApiWorldGenerationStep;
 import com.seibel.lod.common.wrappers.world.ServerLevelWrapper;
+import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.*;
 import com.seibel.lod.core.dataObjects.transformers.FullDataToRenderDataTransformer;
 import com.seibel.lod.core.level.IDhServerLevel;
 import com.seibel.lod.core.config.Config;
@@ -36,6 +37,7 @@ import com.seibel.lod.core.util.objects.LodThreadFactory;
 import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper;
 import com.seibel.lod.core.wrapperInterfaces.worldGeneration.AbstractBatchGenerationEnvironmentWrapper;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -44,14 +46,11 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
 import com.seibel.lod.common.wrappers.DependencySetupDoneCheck;
 import com.seibel.lod.common.wrappers.chunk.ChunkWrapper;
-import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.ChunkLoader;
-import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.LightGetterAdaptor;
-import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.LightedWorldGenRegion;
-import com.seibel.lod.common.wrappers.worldGeneration.mimicObject.WorldGenLevelLightEngine;
 import com.seibel.lod.common.wrappers.worldGeneration.step.StepBiomes;
 import com.seibel.lod.common.wrappers.worldGeneration.step.StepFeatures;
 import com.seibel.lod.common.wrappers.worldGeneration.step.StepLight;
@@ -68,6 +67,8 @@ import net.minecraft.world.level.chunk.ChunkStatus;
 import net.minecraft.world.level.chunk.LevelChunk;
 import net.minecraft.world.level.chunk.ProtoChunk;
 import net.minecraft.world.level.chunk.UpgradeData;
+import net.minecraft.world.level.chunk.storage.RegionFile;
+import net.minecraft.world.level.chunk.storage.RegionFileStorage;
 import net.minecraft.world.level.levelgen.DebugLevelSource;
 import net.minecraft.world.level.levelgen.FlatLevelSource;
 import net.minecraft.world.level.levelgen.NoiseBasedChunkGenerator;
@@ -76,6 +77,8 @@ import net.minecraft.nbt.CompoundTag;
 import net.minecraft.world.level.lighting.LevelLightEngine;
 import org.apache.logging.log4j.LogManager;
 
+import javax.annotation.Nullable;
+
 /*
 Total:                   3.135214124s
 =====================================
@@ -176,6 +179,19 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv
 	public static final int RANGE_TO_RANGE_EMPTY_EXTENSION = 1;
 	public int unknownExceptionCount = 0;
 	public long lastExceptionTriggerTime = 0;
+
+	private AtomicReference<RegionFileStorageExternalCache> regionFileStorageCache = new AtomicReference<>();
+
+	public RegionFileStorageExternalCache getOrCreateRegionFileCache(RegionFileStorage storage) {
+		RegionFileStorageExternalCache cache = regionFileStorageCache.get();
+		if (cache == null) {
+			cache = new RegionFileStorageExternalCache(storage);
+			if (!regionFileStorageCache.compareAndSet(null, cache)) {
+				cache = regionFileStorageCache.get();
+			}
+		}
+		return cache;
+	}
 	
 	public static final LodThreadFactory threadFactory = new LodThreadFactory("DH-Gen-Worker-Thread", Thread.MIN_PRIORITY);
 	
@@ -299,9 +315,11 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv
 			Config.Client.WorldGenerator.enableDistantGeneration.set(false);
 		}
 	}
-	
-	public static ChunkAccess loadOrMakeChunk(ChunkPos chunkPos, ServerLevel level, LevelLightEngine lightEngine)
+
+	public ChunkAccess loadOrMakeChunk(ChunkPos chunkPos, WorldGenLevelLightEngine lightEngine)
 	{
+		ServerLevel level = params.level;
+
 		CompoundTag chunkData = null;
 		try
 		{
@@ -310,7 +328,10 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv
 			#else
 			// Warning: if multiple threads attempt to access this method at the same time,
 			// it can throw EOFExceptions that are caught and logged by Minecraft
-			chunkData = level.getChunkSource().chunkMap.readChunk(chunkPos);
+			//chunkData = level.getChunkSource().chunkMap.readChunk(chunkPos);
+			RegionFileStorage storage = params.level.getChunkSource().chunkMap.worker.storage;
+			RegionFileStorageExternalCache cache = getOrCreateRegionFileCache(storage);
+			chunkData = cache.read(chunkPos);
 			#endif
 		}
 		catch (Exception e)
@@ -329,6 +350,7 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv
 		{
 			try
 			{
+				LOAD_LOGGER.info("DistantHorizons: Loading chunk "+chunkPos+" from disk.");
 				return ChunkLoader.read(level, lightEngine, chunkPos, chunkData);
 			}
 			catch (Exception e)
@@ -367,7 +389,7 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv
 				ChunkAccess target = null;
 				try
 				{
-					target = loadOrMakeChunk(chunkPos, params.level, lightEngine);
+					target = loadOrMakeChunk(chunkPos, lightEngine);
 				}
 				catch (RuntimeException e2)
 				{
@@ -607,7 +629,18 @@ public final class BatchGenerationEnvironment extends AbstractBatchGenerationEnv
 				EVENT_LOGGER.error("Batch Chunk Generator shutdown interrupted! Ignoring child threads...", e);
 			}
 		}
-		
+		var r = regionFileStorageCache.get();
+		if (r != null)
+		{
+			try
+			{
+				r.close();
+			}
+			catch (IOException e)
+			{
+				EVENT_LOGGER.error("Failed to close region file storage cache!", e);
+			}
+		}
 		EVENT_LOGGER.info(BatchGenerationEnvironment.class.getSimpleName()+" shutdown complete.");
 	}
 	

common/src/main/java/com/seibel/lod/common/wrappers/worldGeneration/mimicObject/RegionFileStorageExternalCache.java

@@ -0,0 +1,92 @@
+package com.seibel.lod.common.wrappers.worldGeneration.mimicObject;
+
+import com.seibel.lod.common.wrappers.worldGeneration.BatchGenerationEnvironment;
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.nbt.NbtIo;
+import net.minecraft.world.level.ChunkPos;
+import net.minecraft.world.level.chunk.storage.RegionFile;
+import net.minecraft.world.level.chunk.storage.RegionFileStorage;
+
+import javax.annotation.Nullable;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+public class RegionFileStorageExternalCache implements AutoCloseable {
+    public final RegionFileStorage storage;
+    public static final int MAX_CACHE_SIZE = 16;
+
+    @Override
+    public void close() throws IOException {
+        RegionFileCache cache;
+        while ((cache = regionFileCache.poll()) != null) {
+            cache.file.close();
+        }
+    }
+
+    static class RegionFileCache {
+        public final long pos;
+        public final RegionFile file;
+
+        public RegionFileCache(long pos, RegionFile file) {
+            this.pos = pos;
+            this.file = file;
+        }
+    }
+
+    public ConcurrentLinkedQueue<RegionFileCache> regionFileCache = new ConcurrentLinkedQueue<>();
+
+    public RegionFileStorageExternalCache(RegionFileStorage storage) {
+        this.storage = storage;
+    }
+
+    @Nullable
+    public RegionFile getRegionFile(ChunkPos pos) throws IOException {
+        long posLong = ChunkPos.asLong(pos.getRegionX(), pos.getRegionZ());
+        RegionFile rFile;
+        // First check our custom cache
+        for (RegionFileCache cache : this.regionFileCache) {
+            if (cache.pos == posLong) return cache.file;
+        }
+
+        // Then check vanilla cache
+        while (true) {
+            try {
+                rFile = this.storage.regionCache.getOrDefault(posLong, null);
+                break;
+            } catch (ArrayIndexOutOfBoundsException e) {
+                BatchGenerationEnvironment.LOAD_LOGGER.warn("Concurrency issue detected when getting region file for chunk at " + pos + ". Retrying...");
+            }
+        }
+        if (rFile != null) return rFile;
+
+        // Otherwise, check if file exist, and if so, add it to the cache
+        Path p = storage.folder;
+        if (!Files.exists(p)) return null;
+        Path rFilePath = p.resolve("r." + pos.getRegionX() + "." + pos.getRegionZ() + ".mca");
+        rFile = new RegionFile(rFilePath, p, false);
+        regionFileCache.add(new RegionFileCache(ChunkPos.asLong(pos.getRegionX(), pos.getRegionZ()), rFile));
+        while (regionFileCache.size() > MAX_CACHE_SIZE) {
+            regionFileCache.poll().file.close();
+        }
+        return rFile;
+    }
+
+
+    @Nullable
+    public CompoundTag read(ChunkPos pos) throws IOException {
+        RegionFile file = getRegionFile(pos);
+        if (file == null) return null;
+
+        try (DataInputStream stream = file.getChunkDataInputStream(pos)) {
+            if (stream == null) return null;
+            return NbtIo.read(stream);
+        }
+        catch (Throwable e) {
+            return null;
+        }
+    }
+
+}

common/src/main/resources/1_18.lod.accesswidener

@@ -33,7 +33,10 @@ accessible method net/minecraft/world/level/lighting/LayerLightEngine queueSecti
 # lod generation from save file
 accessible field net/minecraft/server/level/ChunkMap mainThreadExecutor Lnet/minecraft/util/thread/BlockableEventLoop;
 accessible method net/minecraft/server/level/ChunkMap readChunk (Lnet/minecraft/world/level/ChunkPos;)Lnet/minecraft/nbt/CompoundTag;
-
+accessible field net/minecraft/world/level/chunk/storage/ChunkStorage worker Lnet/minecraft/world/level/chunk/storage/IOWorker;
+accessible field net/minecraft/world/level/chunk/storage/IOWorker storage Lnet/minecraft/world/level/chunk/storage/RegionFileStorage;
+accessible field net/minecraft/world/level/chunk/storage/RegionFileStorage regionCache Lit/unimi/dsi/fastutil/longs/Long2ObjectLinkedOpenHashMap;
+accessible field net/minecraft/world/level/chunk/storage/RegionFileStorage folder Ljava/nio/file/Path;
 
 # grabbing textures
 accessible field net/minecraft/client/renderer/texture/TextureAtlasSprite animatedTexture Lnet/minecraft/client/renderer/texture/TextureAtlasSprite$AnimatedTexture;