wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make post-relog update independent from generation toggle

Browse Source
This commit is contained in:
s809
2023-10-23 23:18:52 +05:00
parent 9daf0c7317
commit 9640169be9
7 changed files with 222 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java
+139 -87
View File
@@ -19,34 +19,45 @@
package com.seibel.distanthorizons.core.file.fullDatafile;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue;
import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
import com.seibel.distanthorizons.core.level.DhLevel;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.multiplayer.client.ClientNetworkState;
import com.seibel.distanthorizons.core.network.exceptions.InvalidLevelException;
import com.seibel.distanthorizons.core.network.exceptions.InvalidSectionPosException;
import com.seibel.distanthorizons.core.network.exceptions.RateLimitedException;
import com.seibel.distanthorizons.core.network.messages.fullData.generation.FullDataSourceRequestMessage;
import com.seibel.distanthorizons.core.network.messages.fullData.generation.FullDataSourceResponseMessage;
import com.seibel.distanthorizons.core.network.messages.fullData.updates.FullDataChangeSummaryRequestMessage;
import com.seibel.distanthorizons.core.network.messages.fullData.updates.FullDataChangeSummaryResponseMessage;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import io.netty.channel.ChannelException;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import javax.annotation.CheckForNull;
import java.awt.*;
import java.io.File;
import java.util.ArrayList;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler
public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler implements IDebugRenderable
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@@ -56,94 +67,16 @@ public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler
private final Set<DhSectionPos> visitedSections = ConcurrentHashMap.newKeySet();
private final ConcurrentMap<DhSectionPos, FullDataMetaFile> sectionsToUpdate = new ConcurrentHashMap<>();
private final AtomicBoolean isUpdating = new AtomicBoolean(false);
private boolean invalidSectionsFound = false;
public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable ClientNetworkState networkState)
{
super(level, saveStructure);
this.networkState = networkState;
}
private final F3Screen.NestedMessage f3Message = new F3Screen.NestedMessage(this::f3Log);
private final AtomicInteger finishedRequests = new AtomicInteger();
private final AtomicInteger failedRequests = new AtomicInteger();
public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride, @Nullable ClientNetworkState networkState)
{
super(level, saveStructure, saveDirOverride);
this.networkState = networkState;
}
private void sendUpdateChecks()
{
assert this.networkState != null;
if (this.invalidSectionsFound)
this.sectionsToUpdate.clear();
if (this.sectionsToUpdate.isEmpty())
return;
if (this.isUpdating.getAndSet(true))
return;
Map<DhSectionPos, Integer> block = sectionsToUpdate.entrySet().stream()
.limit(20)
.collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().baseMetaData.checksum));
for (DhSectionPos pos : block.keySet())
sectionsToUpdate.remove(pos);
Consumer<ChunkSizedFullDataAccessor> chunkDataConsumer = (ChunkSizedFullDataAccessor data) -> {
DhSectionPos pos = data.getSectionPos().convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET);
this.writeChunkDataToFile(new DhSectionPos(pos.getDetailLevel(), pos.getX(), pos.getZ()), data);
};
this.networkState.getClient().sendRequest(new FullDataChangeSummaryRequestMessage(level.getLevelWrapper(), block), FullDataChangeSummaryResponseMessage.class)
.handle((response, throwable) ->
{
try
{
if (throwable != null)
throw throwable;
IWorldGenerationQueue queue = this.worldGenQueueRef.get();
if (queue == null)
return null;
for (DhSectionPos pos : response.changedPosList)
{
queue.submitGenTask(pos, pos.getDetailLevel(), new IWorldGenTaskTracker() {
@Override
public boolean isMemoryAddressValid()
{
return true;
}
@NotNull
@Override
public Consumer<ChunkSizedFullDataAccessor> getChunkDataConsumer()
{
return chunkDataConsumer;
}
});
}
}
catch (InvalidLevelException ignored)
{
// We're too late
}
catch (InvalidSectionPosException e)
{
LOGGER.error("Invalid sections found. Updating will not continue.", e);
invalidSectionsFound = true;
}
catch (Throwable e)
{
LOGGER.error("Error while checking section updates", e);
}
finally
{
this.isUpdating.set(false);
sendUpdateChecks();
}
return null;
});
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue);
}
@Override
@@ -163,11 +96,130 @@ public class RemoteFullDataFileHandler extends GeneratedFullDataFileHandler
pos.forEachChildAtLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL, childPos ->
{
FullDataMetaFile childMetaFile = super.getFileIfExist(childPos);
if (childMetaFile != null && visitedSections.add(childPos))
if (childMetaFile != null && childMetaFile.baseMetaData != null && visitedSections.add(childPos))
sectionsToUpdate.put(childPos, childMetaFile);
});
sendUpdateChecks();
return metaFile;
}
private void sendUpdateChecks()
{
assert this.networkState != null;
if (this.sectionsToUpdate.isEmpty())
return;
if (this.isUpdating.getAndSet(true))
return;
Map<DhSectionPos, Integer> block = sectionsToUpdate.entrySet().stream()
.limit(20)
.collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().baseMetaData.checksum));
for (DhSectionPos pos : block.keySet())
sectionsToUpdate.remove(pos);
this.networkState.getClient().sendRequest(new FullDataChangeSummaryRequestMessage(level.getLevelWrapper(), block), FullDataChangeSummaryResponseMessage.class)
.handle((response, throwable) ->
{
try
{
if (throwable != null)
throw throwable;
IWorldGenerationQueue queue = this.worldGenQueueRef.get();
if (queue == null)
return null;
for (DhSectionPos pos : response.changedPosList)
sendUpdateRequest(pos);
}
catch (InvalidLevelException ignored)
{
// We're too late
}
catch (InvalidSectionPosException ignored)
{
}
catch (Throwable e)
{
LOGGER.error("Error while checking section updates", e);
}
finally
{
this.isUpdating.set(false);
sendUpdateChecks();
}
return null;
});
}
public void sendUpdateRequest(DhSectionPos sectionPos)
{
assert this.networkState != null;
this.networkState.getClient().sendRequest(new FullDataSourceRequestMessage(level.getLevelWrapper(), sectionPos, true), FullDataSourceResponseMessage.class)
.handleAsync((response, throwable) ->
{
try
{
if (throwable != null)
throw throwable;
CompleteFullDataSource fullDataSource = response.getFullDataSource(sectionPos, level);
fullDataSource.splitIntoChunkSizedAccessors(((DhLevel) level)::saveWrites);
response.getFullDataSourceLoader().returnPooledDataSource(fullDataSource);
}
catch (InvalidLevelException ignored)
{
// We're too late
}
catch (ChannelException | RateLimitedException ignored)
{
// Can't bother retrying
this.failedRequests.incrementAndGet();
}
catch (Throwable e)
{
LOGGER.error("Error while fetching full data source", e);
this.failedRequests.incrementAndGet();
}
return null;
});
}
private String[] f3Log()
{
if (this.networkState == null || !this.networkState.config.postRelogUpdateEnabled)
return new String[0];
// These metrics are not precise; Updated sections[2] is within range of 1 rate limit or so
ArrayList<String> lines = new ArrayList<>();
lines.add("Post-relog update ["+level.getLevelWrapper().getDimensionType().getDimensionName()+"]");
lines.add("Visited sections: "+visitedSections.size());
lines.add("Updated sections: "+this.finishedRequests+" / "+(this.sectionsToUpdate.size() + this.finishedRequests.get())+" (failed: "+this.failedRequests+")");
return lines.toArray(new String[0]);
}
@Override
public void debugRender(DebugRenderer r)
{
for (Map.Entry<DhSectionPos, FullDataMetaFile> mapEntry : sectionsToUpdate.entrySet())
{
r.renderBox(new DebugRenderer.Box(mapEntry.getKey(), -32f, 64f, 0.05f, Color.pink));
}
}
@Override
public void close()
{
f3Message.close();
DebugRenderer.unregister(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue);
super.close();
}
}
core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java
+56 -22
View File
@@ -97,29 +97,63 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
{
this.eventSource.registerHandler(FullDataSourceRequestMessage.class, remotePlayerConnectionHandler.currentLevelOnly(this, (msg, serverPlayerState) ->
{
if (serverPlayerState.pendingFullDataRequests.incrementAndGet() > serverPlayerState.config.getFullDataRequestRateLimit())
if (msg.changedOnly)
{
serverPlayerState.pendingFullDataRequests.decrementAndGet();
msg.sendResponse(new RateLimitedException("Max concurrent requests: " + serverPlayerState.config.getFullDataRequestRateLimit()));
return;
}
while (true)
{
IncompleteDataSourceEntry entry = incompleteDataSources.computeIfAbsent(msg.dhSectionPos, pos -> {
IncompleteDataSourceEntry newEntry = new IncompleteDataSourceEntry();
serverside.dataFileHandler.readAsync(msg.dhSectionPos).thenAccept(fullDataSource -> {
newEntry.fullDataSource = fullDataSource;
});
return newEntry;
});
// If this fails, current entry is being drained and need to create another one
if (entry.requestCollectionSemaphore.tryAcquire())
if (!serverPlayerState.config.isPostRelogUpdateEnabled())
{
fullDataRequests.put(msg.futureId, entry);
entry.requestMessages.put(msg.futureId, msg);
entry.requestCollectionSemaphore.release();
break;
msg.sendResponse(new RequestRejectedException("Operation is disabled from config."));
return;
}
FullDataMetaFile metaFile = serverside.dataFileHandler.getFileIfExist(msg.dhSectionPos);
if (metaFile == null)
{
msg.sendResponse(new InvalidSectionPosException("Not generated section pos: "+msg.dhSectionPos));
return;
}
metaFile.getOrLoadCachedDataSourceAsync().thenAccept(source -> {
if (!(source instanceof CompleteFullDataSource))
{
msg.sendResponse(new InvalidSectionPosException("Not generated section pos: "+msg.dhSectionPos));
return;
}
msg.sendResponse(new FullDataSourceResponseMessage((CompleteFullDataSource) source, this));
});
}
else
{
if (!serverPlayerState.config.isDistantGenerationEnabled())
{
msg.sendResponse(new RequestRejectedException("Operation is disabled from config."));
return;
}
if (serverPlayerState.pendingFullDataRequests.incrementAndGet() > serverPlayerState.config.getFullDataRequestRateLimit())
{
serverPlayerState.pendingFullDataRequests.decrementAndGet();
msg.sendResponse(new RateLimitedException("Max concurrent requests: " + serverPlayerState.config.getFullDataRequestRateLimit()));
return;
}
while (true)
{
IncompleteDataSourceEntry entry = incompleteDataSources.computeIfAbsent(msg.dhSectionPos, pos -> {
IncompleteDataSourceEntry newEntry = new IncompleteDataSourceEntry();
serverside.dataFileHandler.readAsync(msg.dhSectionPos).thenAccept(fullDataSource -> {
newEntry.fullDataSource = fullDataSource;
});
return newEntry;
});
// If this fails, current entry is being drained and need to create another one
if (entry.requestCollectionSemaphore.tryAcquire())
{
fullDataRequests.put(msg.futureId, entry);
entry.requestMessages.put(msg.futureId, msg);
entry.requestCollectionSemaphore.release();
break;
}
}
}
}));
@@ -133,7 +167,7 @@ public class DhServerLevel extends DhLevel implements IDhServerLevel
this.eventSource.registerHandler(FullDataChangeSummaryRequestMessage.class, remotePlayerConnectionHandler.currentLevelOnly(this, (msg, serverPlayerState) ->
{
if (!Config.Client.Advanced.Multiplayer.ServerNetworking.enablePostRelogUpdate.get())
if (!serverPlayerState.config.isPostRelogUpdateEnabled())
{
msg.sendResponse(new RequestRejectedException("Operation is disabled from config."));
return;
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/AbstractMultiplayerConfig.java
+2
View File
@@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf;
public abstract class AbstractMultiplayerConfig implements INetworkObject
{
public abstract int getRenderDistanceRadius();
public abstract boolean isDistantGenerationEnabled();
public abstract int getFullDataRequestRateLimit();
public abstract boolean isRealTimeUpdatesEnabled();
public abstract boolean isPostRelogUpdateEnabled();
@@ -14,6 +15,7 @@ public abstract class AbstractMultiplayerConfig implements INetworkObject
public void encode(ByteBuf out)
{
out.writeInt(this.getRenderDistanceRadius());
out.writeBoolean(this.isDistantGenerationEnabled());
out.writeInt(this.getFullDataRequestRateLimit());
out.writeBoolean(this.isRealTimeUpdatesEnabled());
out.writeBoolean(this.isPostRelogUpdateEnabled());
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfig.java
+7
View File
@@ -5,9 +5,14 @@ import io.netty.buffer.ByteBuf;
public class MultiplayerConfig extends AbstractMultiplayerConfig
{
// IMPORTANT: Once you added/removed config fields, modify MultiplayerConfigChangeListener accordingly.
public int renderDistanceRadius = Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get();
@Override public int getRenderDistanceRadius() { return renderDistanceRadius; }
public boolean distantGenerationEnabled = Config.Client.Advanced.WorldGenerator.enableDistantGeneration.get();
@Override public boolean isDistantGenerationEnabled() { return distantGenerationEnabled; }
public int fullDataRequestRateLimit = Config.Client.Advanced.Multiplayer.ServerNetworking.requestRateLimit.get();
@Override public int getFullDataRequestRateLimit() { return fullDataRequestRateLimit; }
@@ -21,6 +26,7 @@ public class MultiplayerConfig extends AbstractMultiplayerConfig
public void decode(ByteBuf in)
{
this.renderDistanceRadius = in.readInt();
this.distantGenerationEnabled = in.readBoolean();
this.fullDataRequestRateLimit = in.readInt();
this.realTimeUpdatesEnabled = in.readBoolean();
this.postRelogUpdateEnabled = in.readBoolean();
@@ -30,6 +36,7 @@ public class MultiplayerConfig extends AbstractMultiplayerConfig
{
return "MultiplayerConfig{" +
"renderDistance=" + renderDistanceRadius +
", distantGenerationEnabled=" + distantGenerationEnabled +
", fullDataRequestRateLimit=" + fullDataRequestRateLimit +
", realTimeUpdatesEnabled=" + realTimeUpdatesEnabled +
", postRelogUpdatesEnabled=" + postRelogUpdateEnabled +
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/config/MultiplayerConfigChangeListener.java
+3
View File
@@ -8,6 +8,7 @@ import java.io.Closeable;
public class MultiplayerConfigChangeListener implements Closeable
{
private final ConfigChangeListener<Integer> renderDistanceRadius;
private final ConfigChangeListener<Boolean> enableDistantGeneration;
private final ConfigChangeListener<Integer> requestRateLimit;
private final ConfigChangeListener<Boolean> enableRealTimeUpdates;
private final ConfigChangeListener<Boolean> enablePostRelogUpdate;
@@ -15,6 +16,7 @@ public class MultiplayerConfigChangeListener implements Closeable
public MultiplayerConfigChangeListener(Runnable runnable)
{
renderDistanceRadius = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius, ignored -> runnable.run());
enableDistantGeneration = new ConfigChangeListener<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration, ignored -> runnable.run());
requestRateLimit = new ConfigChangeListener<>(Config.Client.Advanced.Multiplayer.ServerNetworking.requestRateLimit, ignored -> runnable.run());
enableRealTimeUpdates = new ConfigChangeListener<>(Config.Client.Advanced.Multiplayer.ServerNetworking.enableRealTimeUpdates, ignored -> runnable.run());
enablePostRelogUpdate = new ConfigChangeListener<>(Config.Client.Advanced.Multiplayer.ServerNetworking.enablePostRelogUpdate, ignored -> runnable.run());
@@ -24,6 +26,7 @@ public class MultiplayerConfigChangeListener implements Closeable
public void close()
{
renderDistanceRadius.close();
enableDistantGeneration.close();
requestRateLimit.close();
enableRealTimeUpdates.close();
enablePostRelogUpdate.close();
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/ServersideMultiplayerConfig.java
+6
View File
@@ -18,6 +18,12 @@ public class ServersideMultiplayerConfig extends AbstractMultiplayerConfig
return Math.min(clientConfig.renderDistanceRadius, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get());
}
@Override
public boolean isDistantGenerationEnabled()
{
return clientConfig.distantGenerationEnabled && Config.Client.Advanced.WorldGenerator.enableDistantGeneration.get();
}
@Override
public int getFullDataRequestRateLimit()
{
core/src/main/java/com/seibel/distanthorizons/core/network/messages/fullData/generation/FullDataSourceRequestMessage.java
+9
View File
@@ -31,6 +31,7 @@ public class FullDataSourceRequestMessage extends FutureTrackableNetworkMessage
public DhSectionPos dhSectionPos;
private int levelHashCode;
@Override public int getLevelHashCode() { return levelHashCode; }
public boolean changedOnly;
public FullDataSourceRequestMessage() {}
@@ -40,12 +41,19 @@ public class FullDataSourceRequestMessage extends FutureTrackableNetworkMessage
this.levelHashCode = levelWrapper.getDimensionType().getDimensionName().hashCode();
this.dhSectionPos = dhSectionPos;
}
public FullDataSourceRequestMessage(ILevelWrapper levelWrapper, DhSectionPos dhSectionPos, boolean changedOnly)
{
this(levelWrapper, dhSectionPos);
this.changedOnly = true;
}
@Override
public void encode0(ByteBuf out)
{
out.writeInt(levelHashCode);
dhSectionPos.encode(out);
out.writeBoolean(changedOnly);
}
@Override
@@ -53,6 +61,7 @@ public class FullDataSourceRequestMessage extends FutureTrackableNetworkMessage
{
levelHashCode = in.readInt();
dhSectionPos = INetworkObject.decodeStatic(DhSectionPos.zero(), in);
changedOnly = in.readBoolean();
}
@Override