diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java index bbe04f053..6d5c48750 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java @@ -19,34 +19,45 @@ package com.seibel.distanthorizons.core.file.fullDatafile; +import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; -import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; +import com.seibel.distanthorizons.core.level.DhLevel; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.multiplayer.client.ClientNetworkState; import com.seibel.distanthorizons.core.network.exceptions.InvalidLevelException; import com.seibel.distanthorizons.core.network.exceptions.InvalidSectionPosException; +import com.seibel.distanthorizons.core.network.exceptions.RateLimitedException; +import com.seibel.distanthorizons.core.network.messages.fullData.generation.FullDataSourceRequestMessage; +import com.seibel.distanthorizons.core.network.messages.fullData.generation.FullDataSourceResponseMessage; import com.seibel.distanthorizons.core.network.messages.fullData.updates.FullDataChangeSummaryRequestMessage; import com.seibel.distanthorizons.core.network.messages.fullData.updates.FullDataChangeSummaryResponseMessage; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import io.netty.channel.ChannelException; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.annotation.CheckForNull; +import java.awt.*; import java.io.File; +import java.util.ArrayList; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; -public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler +public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -56,94 +67,16 @@ public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler private final Set visitedSections = ConcurrentHashMap.newKeySet(); private final ConcurrentMap sectionsToUpdate = new ConcurrentHashMap<>(); private final AtomicBoolean isUpdating = new AtomicBoolean(false); - private boolean invalidSectionsFound = false; - public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable ClientNetworkState networkState) - { - super(level, saveStructure); - this.networkState = networkState; - } + private final F3Screen.NestedMessage f3Message = new F3Screen.NestedMessage(this::f3Log); + private final AtomicInteger finishedRequests = new AtomicInteger(); + private final AtomicInteger failedRequests = new AtomicInteger(); + public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride, @Nullable ClientNetworkState networkState) { super(level, saveStructure, saveDirOverride); this.networkState = networkState; - } - - private void sendUpdateChecks() - { - assert this.networkState != null; - - if (this.invalidSectionsFound) - this.sectionsToUpdate.clear(); - - if (this.sectionsToUpdate.isEmpty()) - return; - - if (this.isUpdating.getAndSet(true)) - return; - - Map block = sectionsToUpdate.entrySet().stream() - .limit(20) - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().baseMetaData.checksum)); - for (DhSectionPos pos : block.keySet()) - sectionsToUpdate.remove(pos); - - Consumer chunkDataConsumer = (ChunkSizedFullDataAccessor data) -> { - DhSectionPos pos = data.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); - this.writeChunkDataToFile(new DhSectionPos(pos.getDetailLevel(), pos.getX(), pos.getZ()), data); - }; - - this.networkState.getClient().sendRequest(new FullDataChangeSummaryRequestMessage(level.getLevelWrapper(), block), FullDataChangeSummaryResponseMessage.class) - .handle((response, throwable) -> - { - try - { - if (throwable != null) - throw throwable; - - IWorldGenerationQueue queue = this.worldGenQueueRef.get(); - if (queue == null) - return null; - - for (DhSectionPos pos : response.changedPosList) - { - queue.submitGenTask(pos, pos.getDetailLevel(), new IWorldGenTaskTracker() { - @Override - public boolean isMemoryAddressValid() - { - return true; - } - - @NotNull - @Override - public Consumer getChunkDataConsumer() - { - return chunkDataConsumer; - } - }); - } - } - catch (InvalidLevelException ignored) - { - // We're too late - } - catch (InvalidSectionPosException e) - { - LOGGER.error("Invalid sections found. Updating will not continue.", e); - invalidSectionsFound = true; - } - catch (Throwable e) - { - LOGGER.error("Error while checking section updates", e); - } - finally - { - this.isUpdating.set(false); - sendUpdateChecks(); - } - - return null; - }); + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); } @Override @@ -163,11 +96,130 @@ public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler pos.forEachChildAtLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL, childPos -> { FullDataMetaFile childMetaFile = super.getFileIfExist(childPos); - if (childMetaFile != null && visitedSections.add(childPos)) + if (childMetaFile != null && childMetaFile.baseMetaData != null && visitedSections.add(childPos)) sectionsToUpdate.put(childPos, childMetaFile); }); sendUpdateChecks(); return metaFile; } + + private void sendUpdateChecks() + { + assert this.networkState != null; + + if (this.sectionsToUpdate.isEmpty()) + return; + + if (this.isUpdating.getAndSet(true)) + return; + + Map block = sectionsToUpdate.entrySet().stream() + .limit(20) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().baseMetaData.checksum)); + for (DhSectionPos pos : block.keySet()) + sectionsToUpdate.remove(pos); + + this.networkState.getClient().sendRequest(new FullDataChangeSummaryRequestMessage(level.getLevelWrapper(), block), FullDataChangeSummaryResponseMessage.class) + .handle((response, throwable) -> + { + try + { + if (throwable != null) + throw throwable; + + IWorldGenerationQueue queue = this.worldGenQueueRef.get(); + if (queue == null) + return null; + + for (DhSectionPos pos : response.changedPosList) + sendUpdateRequest(pos); + } + catch (InvalidLevelException ignored) + { + // We're too late + } + catch (InvalidSectionPosException ignored) + { + } + catch (Throwable e) + { + LOGGER.error("Error while checking section updates", e); + } + finally + { + this.isUpdating.set(false); + sendUpdateChecks(); + } + + return null; + }); + } + + public void sendUpdateRequest(DhSectionPos sectionPos) + { + assert this.networkState != null; + + this.networkState.getClient().sendRequest(new FullDataSourceRequestMessage(level.getLevelWrapper(), sectionPos, true), FullDataSourceResponseMessage.class) + .handleAsync((response, throwable) -> + { + try + { + if (throwable != null) + throw throwable; + + CompleteFullDataSource fullDataSource = response.getFullDataSource(sectionPos, level); + + fullDataSource.splitIntoChunkSizedAccessors(((DhLevel) level)::saveWrites); + response.getFullDataSourceLoader().returnPooledDataSource(fullDataSource); + } + catch (InvalidLevelException ignored) + { + // We're too late + } + catch (ChannelException | RateLimitedException ignored) + { + // Can't bother retrying + this.failedRequests.incrementAndGet(); + } + catch (Throwable e) + { + LOGGER.error("Error while fetching full data source", e); + this.failedRequests.incrementAndGet(); + } + + return null; + }); + } + + private String[] f3Log() + { + if (this.networkState == null || !this.networkState.config.postRelogUpdateEnabled) + return new String[0]; + + // These metrics are not precise; Updated sections[2] is within range of 1 rate limit or so + ArrayList lines = new ArrayList<>(); + lines.add("Post-relog update ["+level.getLevelWrapper().getDimensionType().getDimensionName()+"]"); + lines.add("Visited sections: "+visitedSections.size()); + lines.add("Updated sections: "+this.finishedRequests+" / "+(this.sectionsToUpdate.size() + this.finishedRequests.get())+" (failed: "+this.failedRequests+")"); + return lines.toArray(new String[0]); + } + + @Override + public void debugRender(DebugRenderer r) + { + for (Map.Entry mapEntry : sectionsToUpdate.entrySet()) + { + r.renderBox(new DebugRenderer.Box(mapEntry.getKey(), -32f, 64f, 0.05f, Color.pink)); + } + } + + @Override + public void close() + { + f3Message.close(); + DebugRenderer.unregister(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); + super.close(); + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index bcf2bff5c..b19d2fb09 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -97,29 +97,63 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel { this.eventSource.registerHandler(FullDataSourceRequestMessage.class, remotePlayerConnectionHandler.currentLevelOnly(this, (msg, serverPlayerState) -> { - if (serverPlayerState.pendingFullDataRequests.incrementAndGet() > serverPlayerState.config.getFullDataRequestRateLimit()) + if (msg.changedOnly) { - serverPlayerState.pendingFullDataRequests.decrementAndGet(); - msg.sendResponse(new RateLimitedException("Max concurrent requests: " + serverPlayerState.config.getFullDataRequestRateLimit())); - return; - } - - while (true) - { - IncompleteDataSourceEntry entry = incompleteDataSources.computeIfAbsent(msg.dhSectionPos, pos -> { - IncompleteDataSourceEntry newEntry = new IncompleteDataSourceEntry(); - serverside.dataFileHandler.readAsync(msg.dhSectionPos).thenAccept(fullDataSource -> { - newEntry.fullDataSource = fullDataSource; - }); - return newEntry; - }); - // If this fails, current entry is being drained and need to create another one - if (entry.requestCollectionSemaphore.tryAcquire()) + if (!serverPlayerState.config.isPostRelogUpdateEnabled()) { - fullDataRequests.put(msg.futureId, entry); - entry.requestMessages.put(msg.futureId, msg); - entry.requestCollectionSemaphore.release(); - break; + msg.sendResponse(new RequestRejectedException("Operation is disabled from config.")); + return; + } + + FullDataMetaFile metaFile = serverside.dataFileHandler.getFileIfExist(msg.dhSectionPos); + if (metaFile == null) + { + msg.sendResponse(new InvalidSectionPosException("Not generated section pos: "+msg.dhSectionPos)); + return; + } + + metaFile.getOrLoadCachedDataSourceAsync().thenAccept(source -> { + if (!(source instanceof CompleteFullDataSource)) + { + msg.sendResponse(new InvalidSectionPosException("Not generated section pos: "+msg.dhSectionPos)); + return; + } + + msg.sendResponse(new FullDataSourceResponseMessage((CompleteFullDataSource) source, this)); + }); + } + else + { + if (!serverPlayerState.config.isDistantGenerationEnabled()) + { + msg.sendResponse(new RequestRejectedException("Operation is disabled from config.")); + return; + } + + if (serverPlayerState.pendingFullDataRequests.incrementAndGet() > serverPlayerState.config.getFullDataRequestRateLimit()) + { + serverPlayerState.pendingFullDataRequests.decrementAndGet(); + msg.sendResponse(new RateLimitedException("Max concurrent requests: " + serverPlayerState.config.getFullDataRequestRateLimit())); + return; + } + + while (true) + { + IncompleteDataSourceEntry entry = incompleteDataSources.computeIfAbsent(msg.dhSectionPos, pos -> { + IncompleteDataSourceEntry newEntry = new IncompleteDataSourceEntry(); + serverside.dataFileHandler.readAsync(msg.dhSectionPos).thenAccept(fullDataSource -> { + newEntry.fullDataSource = fullDataSource; + }); + return newEntry; + }); + // If this fails, current entry is being drained and need to create another one + if (entry.requestCollectionSemaphore.tryAcquire()) + { + fullDataRequests.put(msg.futureId, entry); + entry.requestMessages.put(msg.futureId, msg); + entry.requestCollectionSemaphore.release(); + break; + } } } })); @@ -133,7 +167,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel this.eventSource.registerHandler(FullDataChangeSummaryRequestMessage.class, remotePlayerConnectionHandler.currentLevelOnly(this, (msg, serverPlayerState) -> { - if (!Config.Client.Advanced.Multiplayer.ServerNetworking.enablePostRelogUpdate.get()) + if (!serverPlayerState.config.isPostRelogUpdateEnabled()) { msg.sendResponse(new RequestRejectedException("Operation is disabled from config.")); return; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/AbstractMultiplayerConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/AbstractMultiplayerConfig.java index b55ffde1d..e686ad0ad 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/AbstractMultiplayerConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/AbstractMultiplayerConfig.java @@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf; public abstract class AbstractMultiplayerConfig implements INetworkObject { public abstract int getRenderDistanceRadius(); + public abstract boolean isDistantGenerationEnabled(); public abstract int getFullDataRequestRateLimit(); public abstract boolean isRealTimeUpdatesEnabled(); public abstract boolean isPostRelogUpdateEnabled(); @@ -14,6 +15,7 @@ public abstract class AbstractMultiplayerConfig implements INetworkObject public void encode(ByteBuf out) { out.writeInt(this.getRenderDistanceRadius()); + out.writeBoolean(this.isDistantGenerationEnabled()); out.writeInt(this.getFullDataRequestRateLimit()); out.writeBoolean(this.isRealTimeUpdatesEnabled()); out.writeBoolean(this.isPostRelogUpdateEnabled()); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfig.java index 3534eb10c..406f892c6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfig.java @@ -5,9 +5,14 @@ import io.netty.buffer.ByteBuf; public class MultiplayerConfig extends AbstractMultiplayerConfig { + // IMPORTANT: Once you added/removed config fields, modify MultiplayerConfigChangeListener accordingly. + public int renderDistanceRadius = Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get(); @Override public int getRenderDistanceRadius() { return renderDistanceRadius; } + public boolean distantGenerationEnabled = Config.Client.Advanced.WorldGenerator.enableDistantGeneration.get(); + @Override public boolean isDistantGenerationEnabled() { return distantGenerationEnabled; } + public int fullDataRequestRateLimit = Config.Client.Advanced.Multiplayer.ServerNetworking.requestRateLimit.get(); @Override public int getFullDataRequestRateLimit() { return fullDataRequestRateLimit; } @@ -21,6 +26,7 @@ public class MultiplayerConfig extends AbstractMultiplayerConfig public void decode(ByteBuf in) { this.renderDistanceRadius = in.readInt(); + this.distantGenerationEnabled = in.readBoolean(); this.fullDataRequestRateLimit = in.readInt(); this.realTimeUpdatesEnabled = in.readBoolean(); this.postRelogUpdateEnabled = in.readBoolean(); @@ -30,6 +36,7 @@ public class MultiplayerConfig extends AbstractMultiplayerConfig { return "MultiplayerConfig{" + "renderDistance=" + renderDistanceRadius + + ", distantGenerationEnabled=" + distantGenerationEnabled + ", fullDataRequestRateLimit=" + fullDataRequestRateLimit + ", realTimeUpdatesEnabled=" + realTimeUpdatesEnabled + ", postRelogUpdatesEnabled=" + postRelogUpdateEnabled + diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfigChangeListener.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfigChangeListener.java index b8a13d7ed..4a5c4804f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfigChangeListener.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfigChangeListener.java @@ -8,6 +8,7 @@ import java.io.Closeable; public class MultiplayerConfigChangeListener implements Closeable { private final ConfigChangeListener renderDistanceRadius; + private final ConfigChangeListener enableDistantGeneration; private final ConfigChangeListener requestRateLimit; private final ConfigChangeListener enableRealTimeUpdates; private final ConfigChangeListener enablePostRelogUpdate; @@ -15,6 +16,7 @@ public class MultiplayerConfigChangeListener implements Closeable public MultiplayerConfigChangeListener(Runnable runnable) { renderDistanceRadius = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius, ignored -> runnable.run()); + enableDistantGeneration = new ConfigChangeListener<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration, ignored -> runnable.run()); requestRateLimit = new ConfigChangeListener<>(Config.Client.Advanced.Multiplayer.ServerNetworking.requestRateLimit, ignored -> runnable.run()); enableRealTimeUpdates = new ConfigChangeListener<>(Config.Client.Advanced.Multiplayer.ServerNetworking.enableRealTimeUpdates, ignored -> runnable.run()); enablePostRelogUpdate = new ConfigChangeListener<>(Config.Client.Advanced.Multiplayer.ServerNetworking.enablePostRelogUpdate, ignored -> runnable.run()); @@ -24,6 +26,7 @@ public class MultiplayerConfigChangeListener implements Closeable public void close() { renderDistanceRadius.close(); + enableDistantGeneration.close(); requestRateLimit.close(); enableRealTimeUpdates.close(); enablePostRelogUpdate.close(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/ServersideMultiplayerConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/ServersideMultiplayerConfig.java index 93187ec72..217eab119 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/ServersideMultiplayerConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/ServersideMultiplayerConfig.java @@ -18,6 +18,12 @@ public class ServersideMultiplayerConfig extends AbstractMultiplayerConfig return Math.min(clientConfig.renderDistanceRadius, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get()); } + @Override + public boolean isDistantGenerationEnabled() + { + return clientConfig.distantGenerationEnabled && Config.Client.Advanced.WorldGenerator.enableDistantGeneration.get(); + } + @Override public int getFullDataRequestRateLimit() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/network/messages/fullData/generation/FullDataSourceRequestMessage.java b/core/src/main/java/com/seibel/distanthorizons/core/network/messages/fullData/generation/FullDataSourceRequestMessage.java index 315981a5b..b253f151a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/network/messages/fullData/generation/FullDataSourceRequestMessage.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/network/messages/fullData/generation/FullDataSourceRequestMessage.java @@ -31,6 +31,7 @@ public class FullDataSourceRequestMessage extends FutureTrackableNetworkMessage public DhSectionPos dhSectionPos; private int levelHashCode; @Override public int getLevelHashCode() { return levelHashCode; } + public boolean changedOnly; public FullDataSourceRequestMessage() {} @@ -40,12 +41,19 @@ public class FullDataSourceRequestMessage extends FutureTrackableNetworkMessage this.levelHashCode = levelWrapper.getDimensionType().getDimensionName().hashCode(); this.dhSectionPos = dhSectionPos; } + + public FullDataSourceRequestMessage(ILevelWrapper levelWrapper, DhSectionPos dhSectionPos, boolean changedOnly) + { + this(levelWrapper, dhSectionPos); + this.changedOnly = true; + } @Override public void encode0(ByteBuf out) { out.writeInt(levelHashCode); dhSectionPos.encode(out); + out.writeBoolean(changedOnly); } @Override @@ -53,6 +61,7 @@ public class FullDataSourceRequestMessage extends FutureTrackableNetworkMessage { levelHashCode = in.readInt(); dhSectionPos = INetworkObject.decodeStatic(DhSectionPos.zero(), in); + changedOnly = in.readBoolean(); } @Override