diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index c4dc658c1..49c3b7a9a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -516,7 +516,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo // this will throw a cast exception if the chunk object array isn't correct IChunkWrapper chunk = SingletonInjector.INSTANCE.get(IWrapperFactory.class).createChunkWrapper(chunkObjectArray); - SharedApi.INSTANCE.applyChunkUpdate(chunk, dhLevel.getLevelWrapper(), true, true); + SharedApi.INSTANCE.applyChunkUpdate(chunk, dhLevel.getLevelWrapper()); return DhApiResult.createSuccess(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index bd5f68fdd..96de8af7f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -256,7 +256,6 @@ public class ClientApi if (world != null) { world.unloadLevel(level); - SharedApi.INSTANCE.clearQueuedChunkUpdates(); ApiEventInjector.INSTANCE.fireAllEvents(DhApiLevelUnloadEvent.class, new DhApiLevelUnloadEvent.EventParam(level)); } else @@ -340,7 +339,7 @@ public class ClientApi if (levelWrapper.equals(level)) { IChunkWrapper chunkWrapper = this.waitingChunkByClientLevelAndPos.get(levelChunkPair); - SharedApi.INSTANCE.chunkLoadEvent(chunkWrapper, levelWrapper); + SharedApi.INSTANCE.applyChunkUpdate(chunkWrapper, levelWrapper); keysToRemove.add(levelChunkPair); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java index 12e0cfec2..8d7c5dd61 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java @@ -101,7 +101,6 @@ public class ServerApi if (serverWorld != null) { serverWorld.unloadLevel(level); - SharedApi.INSTANCE.clearQueuedChunkUpdates(); ApiEventInjector.INSTANCE.fireAllEvents(DhApiLevelUnloadEvent.class, new DhApiLevelUnloadEvent.EventParam(level)); } } @@ -112,8 +111,8 @@ public class ServerApi // chunk modified events // //=======================// - public void serverChunkLoadEvent(IChunkWrapper chunkWrapper, ILevelWrapper level) { SharedApi.INSTANCE.applyChunkUpdate(chunkWrapper, level, false, false); } - public void serverChunkSaveEvent(IChunkWrapper chunkWrapper, ILevelWrapper level) { SharedApi.INSTANCE.applyChunkUpdate(chunkWrapper, level, true, false); } + public void serverChunkLoadEvent(IChunkWrapper chunkWrapper, ILevelWrapper level) { SharedApi.INSTANCE.applyChunkUpdate(chunkWrapper, level); } + public void serverChunkSaveEvent(IChunkWrapper chunkWrapper, ILevelWrapper level) { SharedApi.INSTANCE.applyChunkUpdate(chunkWrapper, level); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java index 51260825a..87a44ff21 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java @@ -24,28 +24,21 @@ import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiWorldUn import com.seibel.distanthorizons.core.Initializer; import com.seibel.distanthorizons.core.api.internal.chunkUpdating.ChunkUpdateData; import com.seibel.distanthorizons.core.api.internal.chunkUpdating.ChunkUpdateQueueManager; -import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.api.internal.chunkUpdating.WorldChunkUpdateManager; import com.seibel.distanthorizons.core.config.eventHandlers.IgnoredDimensionCsvHandler; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.enums.MinecraftTextFormat; -import com.seibel.distanthorizons.core.generation.DhLightingEngine; import com.seibel.distanthorizons.core.level.DhClientLevel; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; -import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.Pair; -import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.world.*; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; -import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftRenderWrapper; -import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftSharedWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector; @@ -64,20 +57,8 @@ public class SharedApi /** will be null on the server-side */ @Nullable private static final IMinecraftRenderWrapper MC_RENDER = SingletonInjector.INSTANCE.get(IMinecraftRenderWrapper.class); - /** will be null on the server-side */ - @Nullable - private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); - private static final IMinecraftSharedWrapper MC_SHARED = SingletonInjector.INSTANCE.get(IMinecraftSharedWrapper.class); - public static final ChunkUpdateQueueManager CHUNK_UPDATE_QUEUE_MANAGER = new ChunkUpdateQueueManager(); - /** - * how many chunks can be queued for updating per thread + player (in multiplayer), - * used to prevent updates from infinitely pilling up if the user flies around extremely fast - */ - public static final int MAX_UPDATING_CHUNK_COUNT_PER_THREAD_AND_PLAYER = 1_000; - - /** how many milliseconds must pass before an overloaded message can be sent in chat or the log */ - public static final int MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE = 30_000; + public static final WorldChunkUpdateManager WORLD_CHUNK_UPDATE_MANAGER = WorldChunkUpdateManager.INSTANCE; // local fariable for quick access @Nullable @@ -88,15 +69,19 @@ public class SharedApi //=============// // constructor // //=============// + //region private SharedApi() { } public static void init() { Initializer.init(); } + //endregion + //===============// // world methods // //===============// + //region public static EWorldEnvironment getEnvironment() { return (currentWorld == null) ? null : currentWorld.environment; } @@ -130,7 +115,7 @@ public class SharedApi // shouldn't be necessary, but if we missed closing one of the connections this should make sure they're all closed AbstractDhRepo.closeAllConnections(); // needs to be closed on world shutdown to clear out un-processed chunks - CHUNK_UPDATE_QUEUE_MANAGER.clear(); + WORLD_CHUNK_UPDATE_MANAGER.clear(); // recommend that the garbage collector cleans up any objects from the old world and thread pools System.gc(); @@ -153,39 +138,44 @@ public class SharedApi @Nullable public static IDhServerWorld tryGetDhServerWorld() { return (currentWorld instanceof IDhServerWorld) ? (IDhServerWorld) currentWorld : null; } + //endregion + //==============// // chunk update // //==============// + //region /** * Used to prevent getting a full chunk from MC if it isn't necessary.
* This is important since asking MC for a chunk is slow and may block the render thread. */ - public static boolean isChunkAtBlockPosAlreadyUpdating(int blockPosX, int blockPosZ) - { return CHUNK_UPDATE_QUEUE_MANAGER.contains(new DhChunkPos(new DhBlockPos2D(blockPosX, blockPosZ))); } + public static boolean isChunkAtBlockPosAlreadyUpdating(ILevelWrapper levelWrapper, int blockPosX, int blockPosZ) + { + ChunkUpdateQueueManager manager = WORLD_CHUNK_UPDATE_MANAGER.getByLevelWrapper(levelWrapper); + if (manager == null) + { + return true; + } + + return manager.contains(new DhChunkPos(new DhBlockPos2D(blockPosX, blockPosZ))); + } - public static boolean isChunkAtChunkPosAlreadyUpdating(int chunkPosX, int chunkPosZ) - { return CHUNK_UPDATE_QUEUE_MANAGER.contains(new DhChunkPos(chunkPosX, chunkPosZ)); } - - /** - * This is often fired when unloading a level. - * This is done to prevent overloading the system when - * rapidly changing dimensions. - * (IE prevent DH from infinitely allocating memory - */ - public void clearQueuedChunkUpdates() { CHUNK_UPDATE_QUEUE_MANAGER.clear(); } - - public int getQueuedChunkUpdateCount() { return CHUNK_UPDATE_QUEUE_MANAGER.getQueuedCount(); } + public static boolean isChunkAtChunkPosAlreadyUpdating(ILevelWrapper levelWrapper, int chunkPosX, int chunkPosZ) + { + ChunkUpdateQueueManager manager = WORLD_CHUNK_UPDATE_MANAGER.getByLevelWrapper(levelWrapper); + if (manager == null) + { + return true; + } + + return manager.contains(new DhChunkPos(chunkPosX, chunkPosZ)); + } - /** handles both block place and break events */ - public void chunkBlockChangedEvent(IChunkWrapper chunk, ILevelWrapper level) { this.applyChunkUpdate(chunk, level, true, false); } - public void chunkLoadEvent(IChunkWrapper chunk, ILevelWrapper level) { this.applyChunkUpdate(chunk, level, true, true); } - - public void applyChunkUpdate(IChunkWrapper chunkWrapper, ILevelWrapper level, boolean canGetNeighboringChunks, boolean newlyLoaded) + public void applyChunkUpdate(IChunkWrapper chunkWrapper, ILevelWrapper levelWrapper) { //===================// // validation checks // @@ -200,11 +190,11 @@ public class SharedApi AbstractDhWorld dhWorld = SharedApi.getAbstractDhWorld(); if (dhWorld == null) { - if (level instanceof IClientLevelWrapper) + if (levelWrapper instanceof IClientLevelWrapper) { // If the client world isn't loaded yet, keep track of which chunks were loaded so we can use them later. // This may happen if the client world and client level load events happen out of order - IClientLevelWrapper clientLevel = (IClientLevelWrapper) level; + IClientLevelWrapper clientLevel = (IClientLevelWrapper) levelWrapper; ClientApi.INSTANCE.waitingChunkByClientLevelAndPos.replace(new Pair<>(clientLevel, chunkWrapper.getChunkPos()), chunkWrapper); } @@ -218,13 +208,13 @@ public class SharedApi } // only continue if the level is loaded - IDhLevel dhLevel = dhWorld.getLevel(level); + IDhLevel dhLevel = dhWorld.getLevel(levelWrapper); if (dhLevel == null) { - if (level instanceof IClientLevelWrapper) + if (levelWrapper instanceof IClientLevelWrapper) { // the client level isn't loaded yet - IClientLevelWrapper clientLevel = (IClientLevelWrapper) level; + IClientLevelWrapper clientLevel = (IClientLevelWrapper) levelWrapper; ClientApi.INSTANCE.waitingChunkByClientLevelAndPos.replace(new Pair<>(clientLevel, chunkWrapper.getChunkPos()), chunkWrapper); } @@ -247,21 +237,23 @@ public class SharedApi return; } - // shouldn't normally happen, but just in case - if (CHUNK_UPDATE_QUEUE_MANAGER.contains(chunkWrapper.getChunkPos())) + ChunkUpdateQueueManager chunkManager = WORLD_CHUNK_UPDATE_MANAGER.getByLevelWrapper(levelWrapper); + // ignore the wrong level wrapper type or + // if the chunk is already queued for handling + if (chunkManager == null + || chunkManager.contains(chunkWrapper.getChunkPos())) { - // TODO this will prevent some LODs from updating across dimensions if multiple levels are loaded return; } - queueChunkUpdate(chunkWrapper, dhLevel); + queueChunkUpdate(chunkManager, chunkWrapper, dhLevel); } - private static void queueChunkUpdate(IChunkWrapper chunkWrapper, IDhLevel dhLevel) + private static void queueChunkUpdate(ChunkUpdateQueueManager chunkManager, IChunkWrapper chunkWrapper, IDhLevel dhLevel) { // return if the chunk is already queued - if (CHUNK_UPDATE_QUEUE_MANAGER.contains(chunkWrapper.getChunkPos())) + if (chunkManager.contains(chunkWrapper.getChunkPos())) { return; } @@ -269,193 +261,36 @@ public class SharedApi // add chunk update data to preUpdate queue ChunkUpdateData updateData = new ChunkUpdateData(chunkWrapper, dhLevel); - CHUNK_UPDATE_QUEUE_MANAGER.addItemToPreUpdateQueue(chunkWrapper.getChunkPos(), updateData); + chunkManager.addItemToPreUpdateQueue(chunkWrapper.getChunkPos(), updateData); - // queue updates up to the number of CPU cores allocated for the job - // (this prevents doing extra work queuing tasks that may not be necessary) - // and makes sure the chunks closest to the player are updated first - PriorityTaskPicker.Executor executor = ThreadPoolUtil.getChunkToLodBuilderExecutor(); - if (executor != null - && executor.getQueueSize() < executor.getPoolSize()) - { - try - { - executor.execute(SharedApi::processQueue); - } - catch (RejectedExecutionException ignore) - { - // the executor was shut down, it should be back up shortly and able to accept new jobs - } - } - } - - private static void processQueue() - { - // update the center & max size of the queue manager - int maxUpdateSizeMultiplier; - if (MC_CLIENT != null && MC_CLIENT.playerExists()) - { - // Local worlds & multiplayer - CHUNK_UPDATE_QUEUE_MANAGER.setCenter(MC_CLIENT.getPlayerChunkPos()); - maxUpdateSizeMultiplier = MC_CLIENT.clientConnectedToDedicatedServer() ? 1 : MC_SHARED.getPlayerCount(); - } - else - { - // Dedicated servers - // Also includes spawn chunks since they're likely to be intentionally utilized with updates - maxUpdateSizeMultiplier = 1 + MC_SHARED.getPlayerCount(); - } - - CHUNK_UPDATE_QUEUE_MANAGER.maxSize = MAX_UPDATING_CHUNK_COUNT_PER_THREAD_AND_PLAYER - * Config.Common.MultiThreading.numberOfThreads.get() - * maxUpdateSizeMultiplier; - - - - //===============================// - // update the necessary chunk(s) // - //===============================// - - processQueuedChunkPreUpdate(); - processQueuedChunkUpdate(); - // queue the next position if there are still positions to process AbstractExecutorService executor = ThreadPoolUtil.getChunkToLodBuilderExecutor(); - if (executor != null && !CHUNK_UPDATE_QUEUE_MANAGER.isEmpty()) + if (executor != null) { try { - executor.execute(SharedApi::processQueue); + executor.execute(WORLD_CHUNK_UPDATE_MANAGER::processEachQueue); } catch (RejectedExecutionException ignore) { // the executor was shut down, it should be back up shortly and able to accept new jobs } } - } - private static void processQueuedChunkPreUpdate() - { - ChunkUpdateData preUpdateData = CHUNK_UPDATE_QUEUE_MANAGER.preUpdateQueue.popClosest(); - if (preUpdateData == null) - { - return; - } - - IDhLevel dhLevel = preUpdateData.dhLevel; - IChunkWrapper chunkWrapper = preUpdateData.chunkWrapper; - chunkWrapper.createDhHeightMaps(); - - try - { - // check if this chunk has been converted into an LOD already - boolean checkChunkHash = !Config.Common.LodBuilding.disableUnchangedChunkCheck.get(); - if (checkChunkHash) - { - int oldChunkHash = dhLevel.getChunkHash(chunkWrapper.getChunkPos()); // shouldn't happen on the render thread since it may take a few moments to run - int newChunkHash = chunkWrapper.getBlockBiomeHashCode(); - - boolean hasNewChunkHash = (oldChunkHash != newChunkHash); - if (!hasNewChunkHash) - { - // do not update the chunk if the hash is the same - return; - } - } - - CHUNK_UPDATE_QUEUE_MANAGER.addItemToUpdateQueue(chunkWrapper.getChunkPos(), preUpdateData); - } - catch (Exception e) - { - LOGGER.error("Unexpected error when pre-updating chunk at pos: [" + chunkWrapper.getChunkPos() + "]", e); - } - } - - private static void processQueuedChunkUpdate() - { - ChunkUpdateData updateData = CHUNK_UPDATE_QUEUE_MANAGER.updateQueue.popClosest(); - if (updateData == null) - { - return; - } - - IChunkWrapper chunkWrapper = updateData.chunkWrapper; - IDhLevel dhLevel = updateData.dhLevel; - ILevelWrapper levelWrapper = dhLevel.getLevelWrapper(); - - // having a list of the nearby chunks is needed for lighting and beacon generation - ArrayList nearbyChunkList = tryGetNeighborChunkListForChunk(chunkWrapper); - - - - try - { - // sky lighting is populated later at the data source level - DhLightingEngine.INSTANCE.bakeChunkBlockLighting(chunkWrapper, nearbyChunkList, levelWrapper.hasSkyLight() ? LodUtil.MAX_MC_LIGHT : LodUtil.MIN_MC_LIGHT); - - dhLevel.updateBeaconBeamsForChunk(chunkWrapper, nearbyChunkList); - - int newChunkHash = chunkWrapper.getBlockBiomeHashCode(); - dhLevel.updateChunkAsync(chunkWrapper, newChunkHash); - } - catch (Exception e) - { - LOGGER.error("Unexpected error when updating chunk at pos: [" + chunkWrapper.getChunkPos() + "]", e); - } - - CHUNK_UPDATE_QUEUE_MANAGER.queuedChunkWrapperByChunkPos.remove(updateData.chunkWrapper.getChunkPos()); - } - private static ArrayList tryGetNeighborChunkListForChunk(IChunkWrapper chunkWrapper) - { - // get the neighboring chunk list - ArrayList neighborChunkList = new ArrayList<>(9); - for (int xOffset = -1; xOffset <= 1; xOffset++) - { - for (int zOffset = -1; zOffset <= 1; zOffset++) - { - if (xOffset == 0 && zOffset == 0) - { - // center chunk - neighborChunkList.add(chunkWrapper); - } - else - { - // neighboring chunk - DhChunkPos neighborPos = new DhChunkPos(chunkWrapper.getChunkPos().getX() + xOffset, chunkWrapper.getChunkPos().getZ() + zOffset); - IChunkWrapper neighborChunk = CHUNK_UPDATE_QUEUE_MANAGER.tryGetChunk(neighborPos); - if (neighborChunk != null) - { - neighborChunkList.add(neighborChunk); - } - } - } - } - return neighborChunkList; - } + //endregion //=========// // F3 Menu // //=========// + //region - public String getDebugMenuString() - { - String y = MinecraftTextFormat.YELLOW; - String o = MinecraftTextFormat.ORANGE; - String cf = MinecraftTextFormat.CLEAR_FORMATTING; - - - String preUpdatingCountStr = F3Screen.NUMBER_FORMAT.format(CHUNK_UPDATE_QUEUE_MANAGER.preUpdateQueue.getQueuedCount()); - String updatingCountStr = F3Screen.NUMBER_FORMAT.format(CHUNK_UPDATE_QUEUE_MANAGER.updateQueue.getQueuedCount()); - String queuedCountStr = F3Screen.NUMBER_FORMAT.format(CHUNK_UPDATE_QUEUE_MANAGER.getQueuedCount()); - - String maxUpdateCountStr = F3Screen.NUMBER_FORMAT.format(CHUNK_UPDATE_QUEUE_MANAGER.maxSize); - - return "Queued chunk updates: "+"("+y+preUpdatingCountStr+cf+" + "+o+updatingCountStr+cf+") ["+queuedCountStr+"/"+maxUpdateCountStr+"]"; - } + public ArrayList getDebugMenuString() { return WORLD_CHUNK_UPDATE_MANAGER.getDebugMenuString(); } + + //endregion diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/chunkUpdating/ChunkUpdateQueueManager.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/chunkUpdating/ChunkUpdateQueueManager.java index 7f8dbf416..b45e8f026 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/chunkUpdating/ChunkUpdateQueueManager.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/chunkUpdating/ChunkUpdateQueueManager.java @@ -4,28 +4,58 @@ import com.google.common.cache.CacheBuilder; import com.seibel.distanthorizons.core.api.internal.ClientApi; import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.MinecraftTextFormat; +import com.seibel.distanthorizons.core.generation.DhLightingEngine; +import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.world.EWorldEnvironment; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftSharedWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; import java.util.Collections; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; + +/** + * @see WorldChunkUpdateManager + */ public class ChunkUpdateQueueManager { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); + private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); + private static final IMinecraftSharedWrapper MC_SHARED = SingletonInjector.INSTANCE.get(IMinecraftSharedWrapper.class); + + /** + * how many chunks can be queued for updating per thread + player (in multiplayer), + * used to prevent updates from infinitely pilling up if the user flies around extremely fast + */ + public static final int MAX_UPDATING_CHUNK_COUNT_PER_THREAD_AND_PLAYER = 1_000; + + /** how many milliseconds must pass before an overloaded message can be sent in chat or the log */ + public static final int MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE = 30_000; + + + + private final Set ignoredChunkPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private static long lastOverloadedLogMessageMsTime = 0; + + + public final ChunkPosQueue updateQueue; public final ChunkPosQueue preUpdateQueue; - public final Set ignoredChunkPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); public final ConcurrentMap queuedChunkWrapperByChunkPos = CacheBuilder.newBuilder() .expireAfterWrite(20, TimeUnit.SECONDS) @@ -35,13 +65,15 @@ public class ChunkUpdateQueueManager /** dynamically changes based on the number of threads currently available */ public int maxSize = 500; - private static long lastOverloadedLogMessageMsTime = 0; + /** used to prevent flickering */ + public long lastMsTimeShownActiveInF3Screen = System.currentTimeMillis(); //=============// // constructor // //=============// + //region public ChunkUpdateQueueManager() { @@ -49,11 +81,14 @@ public class ChunkUpdateQueueManager this.preUpdateQueue = new ChunkPosQueue(); } + //endregion + //==================// // list/set methods // //==================// + //region public boolean contains(DhChunkPos pos) { @@ -69,7 +104,8 @@ public class ChunkUpdateQueueManager this.ignoredChunkPosSet.clear(); } public int getQueuedCount() { return this.updateQueue.getQueuedCount() + this.preUpdateQueue.getQueuedCount(); } - public boolean isEmpty() + + public boolean updateQueuesEmpty() { return this.updateQueue.isEmpty() && this.preUpdateQueue.isEmpty(); @@ -114,14 +150,14 @@ public class ChunkUpdateQueueManager { // limit how often an overloaded message can be sent long msBetweenLastLog = System.currentTimeMillis() - lastOverloadedLogMessageMsTime; - if (msBetweenLastLog >= SharedApi.MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE) + if (msBetweenLastLog >= MIN_MS_BETWEEN_OVERLOADED_LOG_MESSAGE) { lastOverloadedLogMessageMsTime = System.currentTimeMillis(); String message = MinecraftTextFormat.ORANGE + "Distant Horizons overloaded, too many chunks queued for LOD processing. " + MinecraftTextFormat.CLEAR_FORMATTING + "\nThis may result in holes in your LODs. " + "\nFix: move through the world slower, decrease your vanilla render distance, slow down your world pre-generator (IE Chunky), or increase the Distant Horizons' CPU thread counts. " + - "\nMax queue count [" + SharedApi.CHUNK_UPDATE_QUEUE_MANAGER.maxSize + "] ([" + SharedApi.MAX_UPDATING_CHUNK_COUNT_PER_THREAD_AND_PLAYER + "] per thread+players)."; + "\nMax queue count [" + this.maxSize + "] ([" + MAX_UPDATING_CHUNK_COUNT_PER_THREAD_AND_PLAYER + "] per thread+players)."; boolean showWarningInChat = Config.Common.Logging.Warning.showUpdateQueueOverloadedChatWarning.get(); if (showWarningInChat) @@ -159,20 +195,180 @@ public class ChunkUpdateQueueManager return existingWrapper.copy(); } + //endregion + //=========// // ignores // //=========// + //region public void addPosToIgnore(DhChunkPos chunkPos) { this.ignoredChunkPosSet.add(chunkPos); } public void removePosToIgnore(DhChunkPos chunkPos) { this.ignoredChunkPosSet.remove(chunkPos); } + //endregion + + + + //===================// + // update processing // + //===================// + //region + + public void processQueue() + { + // update the center & max size of the queue manager + int maxUpdateSizeMultiplier; + if (MC_CLIENT != null && MC_CLIENT.playerExists()) + { + // Local worlds & multiplayer + this.setCenter(MC_CLIENT.getPlayerChunkPos()); + maxUpdateSizeMultiplier = MC_CLIENT.clientConnectedToDedicatedServer() ? 1 : MC_SHARED.getPlayerCount(); + } + else + { + // Dedicated servers + // Also includes spawn chunks since they're likely to be intentionally utilized with updates + maxUpdateSizeMultiplier = 1 + MC_SHARED.getPlayerCount(); + } + + this.maxSize = MAX_UPDATING_CHUNK_COUNT_PER_THREAD_AND_PLAYER + * Config.Common.MultiThreading.numberOfThreads.get() + * maxUpdateSizeMultiplier; + + + + //===============================// + // update the necessary chunk(s) // + //===============================// + + this.processQueuedChunkPreUpdate(); + this.processQueuedChunkUpdate(); + + // queue the next position if there are still positions to process + AbstractExecutorService executor = ThreadPoolUtil.getChunkToLodBuilderExecutor(); + if (executor != null && !this.updateQueuesEmpty()) + { + try + { + executor.execute(this::processQueue); + } + catch (RejectedExecutionException ignore) + { + // the executor was shut down, it should be back up shortly and able to accept new jobs + } + } + + } + + private void processQueuedChunkPreUpdate() + { + ChunkUpdateData preUpdateData = this.preUpdateQueue.popClosest(); + if (preUpdateData == null) + { + return; + } + + IDhLevel dhLevel = preUpdateData.dhLevel; + IChunkWrapper chunkWrapper = preUpdateData.chunkWrapper; + chunkWrapper.createDhHeightMaps(); + + try + { + // check if this chunk has been converted into an LOD already + boolean checkChunkHash = !Config.Common.LodBuilding.disableUnchangedChunkCheck.get(); + if (checkChunkHash) + { + int oldChunkHash = dhLevel.getChunkHash(chunkWrapper.getChunkPos()); // shouldn't happen on the render thread since it may take a few moments to run + int newChunkHash = chunkWrapper.getBlockBiomeHashCode(); + + boolean hasNewChunkHash = (oldChunkHash != newChunkHash); + if (!hasNewChunkHash) + { + // do not update the chunk if the hash is the same + return; + } + } + + this.addItemToUpdateQueue(chunkWrapper.getChunkPos(), preUpdateData); + } + catch (Exception e) + { + LOGGER.error("Unexpected error when pre-updating chunk at pos: [" + chunkWrapper.getChunkPos() + "]", e); + } + } + + private void processQueuedChunkUpdate() + { + ChunkUpdateData updateData = this.updateQueue.popClosest(); + if (updateData == null) + { + return; + } + + IChunkWrapper chunkWrapper = updateData.chunkWrapper; + IDhLevel dhLevel = updateData.dhLevel; + ILevelWrapper levelWrapper = dhLevel.getLevelWrapper(); + + // having a list of the nearby chunks is needed for lighting and beacon generation + ArrayList nearbyChunkList = this.tryGetNeighborChunkListForChunk(chunkWrapper); + + + + try + { + // sky lighting is populated later at the data source level + DhLightingEngine.INSTANCE.bakeChunkBlockLighting(chunkWrapper, nearbyChunkList, levelWrapper.hasSkyLight() ? LodUtil.MAX_MC_LIGHT : LodUtil.MIN_MC_LIGHT); + + dhLevel.updateBeaconBeamsForChunk(chunkWrapper, nearbyChunkList); + + int newChunkHash = chunkWrapper.getBlockBiomeHashCode(); + dhLevel.updateChunkAsync(chunkWrapper, newChunkHash); + } + catch (Exception e) + { + LOGGER.error("Unexpected error when updating chunk at pos: [" + chunkWrapper.getChunkPos() + "]", e); + } + + this.queuedChunkWrapperByChunkPos.remove(updateData.chunkWrapper.getChunkPos()); + } + private ArrayList tryGetNeighborChunkListForChunk(IChunkWrapper chunkWrapper) + { + // get the neighboring chunk list + ArrayList neighborChunkList = new ArrayList<>(9); + for (int xOffset = -1; xOffset <= 1; xOffset++) + { + for (int zOffset = -1; zOffset <= 1; zOffset++) + { + if (xOffset == 0 && zOffset == 0) + { + // center chunk + neighborChunkList.add(chunkWrapper); + } + else + { + // neighboring chunk + DhChunkPos neighborPos = new DhChunkPos(chunkWrapper.getChunkPos().getX() + xOffset, chunkWrapper.getChunkPos().getZ() + zOffset); + IChunkWrapper neighborChunk = this.tryGetChunk(neighborPos); + if (neighborChunk != null) + { + neighborChunkList.add(neighborChunk); + } + } + } + } + return neighborChunkList; + } + + //endregion + //==================// // position methods // //==================// + //region public void setCenter(DhChunkPos newCenter) { @@ -180,5 +376,33 @@ public class ChunkUpdateQueueManager this.preUpdateQueue.setCenter(newCenter); } + //endregion + + + + //=========// + // F3 Menu // + //=========// + //region + + public String getDebugMenuString() + { + String y = MinecraftTextFormat.YELLOW; + String o = MinecraftTextFormat.ORANGE; + String cf = MinecraftTextFormat.CLEAR_FORMATTING; + + + String preUpdatingCountStr = F3Screen.NUMBER_FORMAT.format(this.preUpdateQueue.getQueuedCount()); + String updatingCountStr = F3Screen.NUMBER_FORMAT.format(this.updateQueue.getQueuedCount()); + String queuedCountStr = F3Screen.NUMBER_FORMAT.format(this.getQueuedCount()); + + String maxUpdateCountStr = F3Screen.NUMBER_FORMAT.format(this.maxSize); + + return "Queued chunk updates: "+"("+y+preUpdatingCountStr+cf+" + "+o+updatingCountStr+cf+") ["+queuedCountStr+"/"+maxUpdateCountStr+"]"; + } + + //endregion + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/chunkUpdating/WorldChunkUpdateManager.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/chunkUpdating/WorldChunkUpdateManager.java new file mode 100644 index 000000000..4a1cfc54b --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/chunkUpdating/WorldChunkUpdateManager.java @@ -0,0 +1,180 @@ +package com.seibel.distanthorizons.core.api.internal.chunkUpdating; + +import com.seibel.distanthorizons.core.api.internal.SharedApi; +import com.seibel.distanthorizons.core.logging.DhLogger; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.world.AbstractDhWorld; +import com.seibel.distanthorizons.core.world.EWorldEnvironment; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Holds all the {@link ChunkUpdateQueueManager} for a loaded world. + * Different queues are needed for each level to prevent + * chunks from bleeding between levels (IE a nether chunk applied to the overworld). + * + * @see ChunkUpdateQueueManager + */ +public class WorldChunkUpdateManager +{ + private static final DhLogger LOGGER = new DhLoggerBuilder().build(); + + /** singleton since we only expect to have one world loaded at a time */ + public static final WorldChunkUpdateManager INSTANCE = new WorldChunkUpdateManager(); + + /** + * Queues are only removed during world shutdown. + * The assumption is that there will be a limited number of {@link ILevelWrapper}'s + * for a given world. + */ + private final ConcurrentHashMap updateQueueByLevelWrapper = new ConcurrentHashMap<>(); + + + + //=============// + // constructor // + //=============// + //region + + private WorldChunkUpdateManager() { } + + //endregion + + + + //=================// + // manager methods // + //=================// + //region + + /** + * @return null if the world is unloaded or the given level wrapper is the wrong type + */ + @Nullable + public ChunkUpdateQueueManager getByLevelWrapper(ILevelWrapper levelWrapper) + { + AbstractDhWorld world = SharedApi.getAbstractDhWorld(); + if (world == null) + { + return null; + } + + // we only want to load chunks for certain level wrappers + // this is done specifically on a local-server to prevent + // loading both the server and client level wrappers + if (world.environment == EWorldEnvironment.CLIENT_ONLY + // when connected to a server we should only ever load client wrappers anyway + // but this check confirms it + && !(levelWrapper instanceof IClientLevelWrapper)) + { + return null; + } + else if ( + (world.environment == EWorldEnvironment.SERVER_ONLY + || world.environment == EWorldEnvironment.CLIENT_SERVER) + // when hosting a server we only care about the server wrappers + && !(levelWrapper instanceof IServerLevelWrapper)) + { + return null; + } + + + ChunkUpdateQueueManager queueManager = this.updateQueueByLevelWrapper.get(levelWrapper); + if (queueManager != null) + { + return queueManager; + } + + return this.updateQueueByLevelWrapper.compute(levelWrapper, + (ILevelWrapper newLevelWrapper, ChunkUpdateQueueManager oldQueueManager) -> + { + if (oldQueueManager != null) + { + return oldQueueManager; + } + + oldQueueManager = new ChunkUpdateQueueManager(); + return oldQueueManager; + }); + } + + public void processEachQueue() + { + this.updateQueueByLevelWrapper.forEach( + (ILevelWrapper levelWrapper, ChunkUpdateQueueManager updateManager) -> + { + updateManager.processQueue(); + }); + } + + public int getTotalQueuedCount() + { + AtomicInteger queueCountRef = new AtomicInteger(0); + + this.updateQueueByLevelWrapper.forEach( + (ILevelWrapper levelWrapper, ChunkUpdateQueueManager updateManager) -> + { + queueCountRef.addAndGet(updateManager.getQueuedCount()); + }); + + return queueCountRef.get(); + } + + public void clear() { this.updateQueueByLevelWrapper.clear(); } + + //endregion + + + + //=========// + // F3 Menu // + //=========// + //region + + public ArrayList getDebugMenuString() + { + ArrayList stringList = new ArrayList<>(); + stringList.add("");// placeholder for the total count + + // add each queue to the list + AtomicInteger totalQueueCountRef = new AtomicInteger(0); + AtomicInteger activeQueueCountRef = new AtomicInteger(0); + this.updateQueueByLevelWrapper.forEach( + (ILevelWrapper levelWrapper, ChunkUpdateQueueManager updateManager) -> + { + // is this queue active? + if (!updateManager.updateQueuesEmpty()) + { + updateManager.lastMsTimeShownActiveInF3Screen = System.currentTimeMillis(); + activeQueueCountRef.incrementAndGet(); + } + + // show this queue if it hasn't been empty long enough + // (done to prevent flickering on the F3 screen when the queue rapidly fills/empties) + long timeSinceQueueLastShownActiveMs = System.currentTimeMillis() - updateManager.lastMsTimeShownActiveInF3Screen; + if (timeSinceQueueLastShownActiveMs < 4_000) + { + stringList.add(levelWrapper.getDimensionName() + ": " + updateManager.getDebugMenuString()); + } + + totalQueueCountRef.incrementAndGet(); + }); + + // replace the first line with the number of total/active queues + // (helpful if we need to diagnose a leak due to a massive number of queue level wrappers) + stringList.set(0, "Chunk Update Queues: "+totalQueueCountRef.get()+"/"+activeQueueCountRef.get()); + + return stringList; + } + + //endregion + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 0b135fb14..81b959b69 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.api.internal.SharedApi; +import com.seibel.distanthorizons.core.api.internal.chunkUpdating.WorldChunkUpdateManager; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; @@ -228,7 +228,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im int maxWorldGenQueueCount = MAX_WORLD_GEN_REQUESTS_PER_THREAD * Config.Common.MultiThreading.numberOfThreads.get(); - int currentQueueCount = SharedApi.INSTANCE.getQueuedChunkUpdateCount(); + int currentQueueCount = WorldChunkUpdateManager.INSTANCE.getTotalQueuedCount(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java b/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java index f28c17a27..25e1a97eb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java @@ -161,7 +161,8 @@ public class F3Screen // chunk updates if (Config.Client.Advanced.Debugging.F3Screen.showQueuedChunkUpdateCount.get()) { - messageList.add(SharedApi.INSTANCE.getDebugMenuString()); + ArrayList chunkQueueList = SharedApi.INSTANCE.getDebugMenuString(); + messageList.addAll(chunkQueueList); messageList.add(""); }