wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve generationQueue and add more and better logging and fix double close on DhLevels

Browse Source
This commit is contained in:
TomTheFurry
2022-07-27 14:49:03 +08:00
parent 30aba99c27
commit 6862f5667b
10 changed files with 164 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/seibel/lod/core/a7/datatype/transform/FullToColumnTransformer.java
+15 -2
View File
@@ -26,7 +26,7 @@ public class FullToColumnTransformer {
final int vertSize = Config.Client.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(data.getDataDetail());
final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY());
if (dataDetail == pos.sectionDetail - columnSource.getDataDetail()) {
if (dataDetail == columnSource.getDataDetail()) {
for (int x = 0; x < pos.getWidth(dataDetail).value; x++) {
for (int z = 0; z < pos.getWidth(dataDetail).value; z++) {
ColumnArrayView columnArrayView = columnSource.getVerticalDataView(x, z);
@@ -34,6 +34,19 @@ public class FullToColumnTransformer {
convertColumnData(level, columnArrayView, fullArrayView);
}
}
// } else if (dataDetail == 0 && columnSource.getDataDetail() > dataDetail) {
// byte deltaDetail = (byte) (columnSource.getDataDetail() - dataDetail);
// int perColumnWidth = 1 << deltaDetail;
// int columnCount = pos.getWidth(dataDetail).value / perColumnWidth;
//
//
// for (int x = 0; x < pos.getWidth(dataDetail).value; x++) {
// for (int z = 0; z < pos.getWidth(dataDetail).value; z++) {
// ColumnArrayView columnArrayView = columnSource.getVerticalDataView(x, z);
// SingleFullArrayView fullArrayView = data.get(x, z);
// convertColumnData(level, columnArrayView, fullArrayView);
// }
// }
} else {
throw new UnsupportedOperationException("To be implemented");
//FIXME: Implement different size creation of renderData
@@ -46,7 +59,7 @@ public class FullToColumnTransformer {
private static void convertColumnData(IClientLevel level, ColumnArrayView columnArrayView, SingleFullArrayView fullArrayView) {
if (!fullArrayView.doesItExist()) return;
// TODO: Set gen mode
int genModeValue = 0;
int genModeValue = 1;
int dataTotalLength = fullArrayView.getSingleLength();
if (dataTotalLength == 0) return;
src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java
+77 -41
View File
@@ -9,11 +9,14 @@ import com.seibel.lod.core.a7.pos.DhLodPos;
import com.seibel.lod.core.a7.pos.DhSectionPos;
import com.seibel.lod.core.logging.DhLoggerBuilder;
import com.seibel.lod.core.objects.DHChunkPos;
import com.seibel.lod.core.util.LodUtil;
import com.seibel.lod.core.util.gridList.ArrayGridList;
import org.apache.logging.log4j.Logger;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
@@ -24,6 +27,7 @@ public class GenerationQueue implements PlaceHolderQueue {
DhBlockPos2D lastPlayerPos = new DhBlockPos2D(0, 0);
final HashMap<DhSectionPos, WeakReference<PlaceHolderRenderSource>> trackers = new HashMap<>();
final BiConsumer<DhSectionPos, ChunkSizedData> writeConsumer;
final HashSet<DhSectionPos> inProgressSections = new HashSet<>();
public GenerationQueue(BiConsumer<DhSectionPos, ChunkSizedData> writeConsumer) {
this.writeConsumer = writeConsumer;
@@ -49,29 +53,25 @@ public class GenerationQueue implements PlaceHolderQueue {
//FIXME: Do optimizations on polling closest to player. (Currently its a O(n) search!)
//FIXME: Do not return sections that is already being generated.
//FIXME: Optimize the checks for inProgressSections.
private DhSectionPos pollClosest(DhBlockPos2D playerPos) {
update();
DhSectionPos closest = null;
long closestDist = Long.MAX_VALUE;
for (DhSectionPos pos : trackers.keySet()) {
if (inProgressSections.contains(pos)) {
continue;
}
long distSqr = pos.getCenter().getCenter().distSquared(playerPos);
if (distSqr < closestDist) {
closest = pos;
closestDist = distSqr;
}
}
if (closest != null) inProgressSections.add(closest);
return closest;
}
private void write(DhSectionPos pos, ChunkSizedData data) {
writeConsumer.accept(pos, data);
WeakReference<PlaceHolderRenderSource> ref = trackers.get(pos);
if (ref == null) return; // No placeholder there, so no need to trigger a refresh on it.
PlaceHolderRenderSource source = ref.get();
if (source == null) return; // Same as above.
source.markInvalid(); // Mark the placeholder as invalid, so it will be refreshed on next lodTree update.
}
public void doGeneration(IGenerator generator) {
if (generator == null) return;
if (generator.isBusy()) return;
@@ -109,40 +109,76 @@ public class GenerationQueue implements PlaceHolderQueue {
assert count > 0;
assert granularity >= 4; // Thanks compiler. Guess having a 'always true' warning means I did it right.
logger.info("Generating section {} of size {} with granularity {} at {}", pos, count, granularity, chunkPosMin);
//FIXME: Handle size != 1 case
CompletableFuture<ArrayGridList<ChunkSizedData>> dataFuture = generator.generate(chunkPosMin, granularity);
int perCallChunksWidth = 1 << (granularity - 4);
final byte sectionDetail = (byte) (dataDetail + FullDataSource.SECTION_SIZE_OFFSET);
dataFuture.whenComplete((data, ex) -> {
if (ex != null) {
if (ex instanceof CompletionException) {
ex = ex.getCause();
}
logger.error("Error generating data for section {}", pos, ex);
return;
ArrayList<CompletableFuture<Void>> futures = new ArrayList<>(count*count);
for (int dx = 0; dx < count; dx++) {
for (int dz = 0; dz < count; dz++) { // TODO: Unroll this loop to yield when generator is busy.
DHChunkPos subCallChunkPosMin = new DHChunkPos(chunkPosMin.x + dx * perCallChunksWidth, chunkPosMin.z + dz * perCallChunksWidth);
CompletableFuture<ArrayGridList<ChunkSizedData>> dataFuture = generator.generate(subCallChunkPosMin, granularity);
futures.add(dataFuture.whenComplete((data, ex) -> {
if (ex != null) {
if (ex instanceof CompletionException) {
ex = ex.getCause();
}
logger.error("Error generating data for section {}", pos, ex);
return;
}
assert data != null;
if (data.gridSize < (1 << (granularity-4))) {
logger.error(
"Generator at {} returned {} by {} chunks but requested granularity was {}, which expect at least {} by {} chunks! ",
pos, data.gridSize, data.gridSize, granularity, perCallChunksWidth, perCallChunksWidth);
return;
}
DhLodPos minSectPos = new DhLodPos((byte)(dataDetail+4), data.getFirst().x, data.getFirst().z).convertUpwardsTo(sectionDetail);
DhLodPos maxSectPos = new DhLodPos((byte)(dataDetail+4), data.getLast().x, data.getLast().z).convertUpwardsTo(sectionDetail);
int sectionCount = (maxSectPos.x - minSectPos.x) + 1;
LodUtil.assertTrue(sectionCount > 0 && sectionCount == (maxSectPos.z - minSectPos.z) + 1);
logger.info("Writing {} by {} chunks (at {}) with data detail {} to {} by {} sections (at {})",
data.gridSize, data.gridSize, subCallChunkPosMin, dataDetail,
sectionCount, sectionCount, minSectPos);
data.forEachPos((x,z) -> {
ChunkSizedData chunkData = data.get(x,z);
DhLodPos chunkDataPos = new DhLodPos((byte)(chunkData.dataDetail + 4), chunkData.x, chunkData.z).convertUpwardsTo(sectionDetail);
DhSectionPos sectionPos = new DhSectionPos(chunkDataPos.detail, chunkDataPos.x, chunkDataPos.z);
//logger.info("Writing chunk {} with data detail {} to section {}",
// new DhLodPos((byte)(chunkData.dataDetail + 4), chunkData.x, chunkData.z),
// dataDetail, sectionPos);
writeConsumer.accept(sectionPos, chunkData);
});
//
// for (int dsx = 0; dsx < sectionCount; dsx++) {
// for (int dsz = 0; dsz < sectionCount; dsz++) {
// WeakReference<PlaceHolderRenderSource> ref = trackers.remove(new DhSectionPos(
// sectionDetail, minSectPos.x + dsx, minSectPos.z + dsz));
// if (ref == null) return; // No placeholder there, so no need to trigger a refresh on it.
// PlaceHolderRenderSource source = ref.get();
// if (source == null) return; // Same as above.
// source.markInvalid(); // Mark the placeholder as invalid, so it will be refreshed on next lodTree update.
// }
// }
}).exceptionally(ex -> {
logger.error("Error generating data for {} by {} chunks (at {}) with data detail {}",
perCallChunksWidth, perCallChunksWidth, subCallChunkPosMin, dataDetail, ex);
return null;
}).thenRun(()->{})); // Convert to a CompletableFuture<Void>.
}
assert data != null;
if (data.gridSize < (1 << (granularity-4)))
throw new IllegalStateException("Generator returned chunks of size "
+ data.gridSize + " but requested granularity was " + granularity
+ " (equals to chunks of : " + (1 << (granularity-4)) + ") @ " + chunkPosMin);
logger.info("Writing chunk {} - {} with data detail {}",
chunkPosMin, new DHChunkPos(chunkPosMin.x + (1 << (granularity-4)), chunkPosMin.z + (1 << (granularity-4))),
dataDetail);
final byte sectionDetail = (byte) (dataDetail + FullDataSource.SECTION_SIZE_OFFSET);
data.forEachPos((x,z) -> {
ChunkSizedData chunkData = data.get(x,z);
DhLodPos chunkDataPos = new DhLodPos((byte)(chunkData.dataDetail + 4), chunkData.x, chunkData.z).convertUpwardsTo(sectionDetail);
DhSectionPos sectionPos = new DhSectionPos(chunkDataPos.detail, chunkDataPos.x, chunkDataPos.z);
logger.info("Writing chunk {} with data detail {} to section {}",
new DhLodPos((byte)(chunkData.dataDetail + 4), chunkData.x, chunkData.z),
dataDetail, sectionPos);
write(sectionPos, chunkData);
});
}).exceptionally(ex -> {
logger.error("Error generating data for section {}", pos, ex);
return null;
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenRun(() -> {
//try {
//Thread.sleep(10000); // FIXME: Only for current debug testing. REMOVE THIS!
//} catch (InterruptedException ignored) {}
WeakReference<PlaceHolderRenderSource> ref = trackers.remove(pos);
if (ref == null) return; // No placeholder there, so no need to trigger a refresh on it.
PlaceHolderRenderSource source = ref.get();
if (source == null) return; // Same as above.
source.markInvalid(); // Mark the placeholder as invalid, so it will be refreshed on next lodTree update.
});
}
src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java
+42 -11
View File
@@ -5,6 +5,7 @@ import java.lang.ref.SoftReference;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
@@ -40,6 +41,7 @@ public class DataMetaFile extends MetaFile {
AtomicReference<GuardedMultiAppendQueue> writeQueue =
new AtomicReference<>(new GuardedMultiAppendQueue());
GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue();
private final AtomicBoolean inCacheWriteLock = new AtomicBoolean(false);
public void addToWriteQueue(ChunkSizedData datatype) {
DhLodPos chunkPos = new DhLodPos((byte) (datatype.dataDetail + 4), datatype.x, datatype.z);
@@ -102,16 +104,37 @@ public class DataMetaFile extends MetaFile {
return isValid;
}
// Suppress casting of CompletableFuture<?> to CompletableFuture<LodDataSource>
@SuppressWarnings("unchecked")
// "unchecked": Suppress casting of CompletableFuture<?> to CompletableFuture<LodDataSource>
// "PointlessBooleanExpression": Suppress explicit (boolean == false) check for more understandable CAS operation code.
@SuppressWarnings({"unchecked", "PointlessBooleanExpression"})
private CompletableFuture<LodDataSource> _readCached(Object obj) {
// Has file cached in RAM and not freed yet.
if ((obj instanceof SoftReference<?>)) {
Object inner = ((SoftReference<?>)obj).get();
if (inner != null) {
LodUtil.assertTrue(inner instanceof LodDataSource);
//TODO: Apply the write if queue is not empty
boolean isEmpty = writeQueue.get().queue.isEmpty();
// If the queue is empty, and the CAS on inCacheWriteLock succeeds, then we are the thread
// that will be applying the changes to the cache.
if (!isEmpty) {
// Do a CAS on inCacheWriteLock to ensure that we are the only thread that is writing to the cache,
// or if we fail, then that means someone else is already doing it, and we can just continue.
// FIXME: Should we return a future that waits for the write to be done for CAS fail? Or should we just return the
// cached data that doesn't have all writes done immediately?
// The latter give us immediate access to the data, but we need to ensure concurrent reads and
// writes doesn't cause unexpected behavior down the line.
// For now, I'll go for the latter option and just hope nothing goes wrong...
if (inCacheWriteLock.getAndSet(true) == false) {
try {
applyWriteQueue((LodDataSource) inner);
} catch (Exception e) {
LOGGER.error("Error while applying changes to LodDataSource at {}: ", pos, e);
} finally {
inCacheWriteLock.set(false);
}
}
}
// Finally, return the cached data.
return CompletableFuture.completedFuture((LodDataSource)inner);
}
}
@@ -151,11 +174,9 @@ public class DataMetaFile extends MetaFile {
});
return future;
}
private LodDataSource loadAndUpdateDataSource() {
LodDataSource data = loadFile();
if (data == null) data = FullDataSource.createEmpty(pos);
// Return whether any write has happened to the data
private void applyWriteQueue(LodDataSource data) {
// Poll the write queue
// First check if write queue is empty, then swap the write queue.
// Must be done in this order to ensure isValid work properly. See isValid() for details.
@@ -164,13 +185,23 @@ public class DataMetaFile extends MetaFile {
if (!isEmpty) {
localVer = localVersion.incrementAndGet();
swapWriteQueue();
int count = _backQueue.queue.size();
for (ChunkSizedData chunk : _backQueue.queue) {
data.update(chunk);
}
write(data);
LOGGER.info("Updated Data file at {} for sect {}", path, pos);
LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count);
} else localVer = localVersion.get();
data.setLocalVersion(localVer);
}
private LodDataSource loadAndUpdateDataSource() {
LodDataSource data = loadFile();
if (data == null) data = FullDataSource.createEmpty(pos);
// Apply the write queue
LodUtil.assertTrue(!inCacheWriteLock.get(),"No one should be writing to the cache while we are in the process of " +
"loading one into the cache! Is this a deadlock?");
applyWriteQueue(data);
// Finally, return the data.
return data;
}
@@ -180,7 +211,7 @@ public class DataMetaFile extends MetaFile {
// Refresh the metadata.
try {
super.updateMetaData();
} catch (IOException e) {
} catch (Exception e) {
LOGGER.warn("Metadata for file {} changed unexpectedly and in an invalid state. Dropping file.", path, e);
return null;
}
@@ -188,7 +219,7 @@ public class DataMetaFile extends MetaFile {
// Load the file.
try (FileInputStream fio = getDataContent()){
return loader.loadData(this, fio, level);
} catch (IOException e) {
} catch (Exception e) {
LOGGER.warn("Failed to load file {}. Dropping file.", path, e);
return null;
}
src/main/java/com/seibel/lod/core/a7/world/DhClientServerWorld.java
+1 -2
View File
@@ -44,8 +44,7 @@ public class DhClientServerWorld extends DhWorld implements IClientWorld, IServe
@Override
public void unloadLevel(ILevelWrapper wrapper) {
if (levels.containsKey(wrapper)) {
LOGGER.info("Unloading level for world " + wrapper.getDimensionType().getDimensionName());
levels.get(wrapper).close();
LOGGER.info("Unloading level {} ", levels.get(wrapper));
levels.remove(wrapper).close();
}
}
src/main/java/com/seibel/lod/core/a7/world/DhClientWorld.java
+1 -2
View File
@@ -45,8 +45,7 @@ public class DhClientWorld extends DhWorld implements IClientWorld {
@Override
public void unloadLevel(ILevelWrapper wrapper) {
if (levels.containsKey(wrapper)) {
LOGGER.info("Unloading level for world " + wrapper.getDimensionType().getDimensionName());
levels.get(wrapper).close();
LOGGER.info("Unloading level {} ", levels.get(wrapper));
levels.remove(wrapper).close();
}
}
src/main/java/com/seibel/lod/core/a7/world/DhServerWorld.java
+1 -2
View File
@@ -37,8 +37,7 @@ public class DhServerWorld extends DhWorld implements IServerWorld {
@Override
public void unloadLevel(ILevelWrapper wrapper) {
if (levels.containsKey(wrapper)) {
LOGGER.info("Unloading level for world " + wrapper.getDimensionType().getDimensionName());
levels.get(wrapper).close();
LOGGER.info("Unloading level {} ", levels.get(wrapper));
levels.remove(wrapper).close();
}
}
src/main/java/com/seibel/lod/core/api/internal/a7/ClientApi.java
+7
View File
@@ -57,6 +57,7 @@ public class ClientApi
{
public static final Logger LOGGER = LogManager.getLogger(ClientApi.class.getSimpleName());
public static boolean prefLoggerEnabled = false;
public static final boolean ENABLE_EVENT_LOGGING = true;
public static final ClientApi INSTANCE = new ClientApi();
public static RenderSystemTest testRenderer = new RenderSystemTest();
@@ -108,9 +109,11 @@ public class ClientApi
}
public void onClientOnlyConnected() {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Client on ClientOnly mode connecting.");
SharedApi.currentWorld = new DhClientWorld();
}
public void onClientOnlyDisconnected() {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Client on ClientOnly mode disconnecting.");
SharedApi.currentWorld.close();
SharedApi.currentWorld = null;
}
@@ -130,6 +133,7 @@ public class ClientApi
public void clientLevelUnloadEvent(ILevelWrapper level)
{
if (ENABLE_EVENT_LOGGING) LOGGER.info("Client level {} unloading.", level);
if (SharedApi.currentWorld instanceof DhClientServerWorld) {
((DhClientServerWorld)SharedApi.currentWorld).disableRendering(level);
} else if (SharedApi.getEnvironment() == WorldEnvironment.Client_Only) {
@@ -138,6 +142,7 @@ public class ClientApi
}
public void clientLevelLoadEvent(ILevelWrapper level)
{
if (ENABLE_EVENT_LOGGING) LOGGER.info("Client level {} loading.", level);
if (SharedApi.currentWorld instanceof DhClientServerWorld) {
((DhClientServerWorld)SharedApi.currentWorld).enableRendering(level);
} else if (SharedApi.getEnvironment() == WorldEnvironment.Client_Only) {
@@ -148,12 +153,14 @@ public class ClientApi
private long lastFlush = 0;
public void rendererShutdownEvent() {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Renderer shutting down.");
IProfilerWrapper profiler = MC.getProfiler();
profiler.push("DH-RendererShutdown");
profiler.pop();
}
public void rendererStartupEvent() {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Renderer starting up.");
IProfilerWrapper profiler = MC.getProfiler();
profiler.push("DH-RendererStartup");
// make sure the GLProxy is created before the LodBufferBuilder needs it
src/main/java/com/seibel/lod/core/api/internal/a7/ServerApi.java
+6
View File
@@ -44,6 +44,7 @@ public class ServerApi
public static final ServerApi INSTANCE = new ServerApi();
private static final Logger LOGGER = DhLoggerBuilder.getLogger(MethodHandles.lookup().lookupClass().getSimpleName());
private static final IVersionConstants VERSION_CONSTANTS = SingletonInjector.INSTANCE.get(IVersionConstants.class);
public static final boolean ENABLE_EVENT_LOGGING = true;
private ServerApi()
{
@@ -69,6 +70,7 @@ public class ServerApi
//TODO: rename to serverLoadEvent
public void serverWorldLoadEvent(boolean isDedicatedEnvironment) {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Server World loading with (dedicated?:{})", isDedicatedEnvironment);
if (isDedicatedEnvironment) {
SharedApi.currentWorld = new DhServerWorld();
} else {
@@ -78,21 +80,25 @@ public class ServerApi
//TODO: rename to serverUnloadEvent
public void serverWorldUnloadEvent() {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Server World {} unloading", SharedApi.currentWorld);
SharedApi.currentWorld.close();
SharedApi.currentWorld = null;
}
public void serverLevelLoadEvent(ILevelWrapper world) {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Server Level {} loading", world);
if (SharedApi.currentWorld instanceof IServerWorld)
SharedApi.currentWorld.getOrLoadLevel(world);
}
public void serverLevelUnloadEvent(ILevelWrapper world) {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Server Level {} unloading", world);
if (SharedApi.currentWorld instanceof IServerWorld)
SharedApi.currentWorld.unloadLevel(world);
}
@Deprecated
public void serverSaveEvent() {
if (ENABLE_EVENT_LOGGING) LOGGER.info("Server world {} saving", SharedApi.currentWorld);
if (SharedApi.currentWorld instanceof IServerWorld)
SharedApi.currentWorld.saveAndFlush();
}
src/main/java/com/seibel/lod/core/objects/DHChunkPos.java
+1 -1
View File
@@ -121,7 +121,7 @@ public class DHChunkPos {
@Override
public String toString() {
return "DHChunkPos[" + x + ", " + z + "]";
return "C[" + x + "," + z + "]";
}
src/main/java/com/seibel/lod/core/util/gridList/ArrayGridList.java
+13
View File
@@ -75,10 +75,23 @@ public class ArrayGridList<T> extends ArrayList<T> {
if (!inRange(x,y)) return null;
return get(_indexOf(x,y));
}
public T getFirst() {
return get(0,0);
}
public T getLast() {
return get(gridSize-1, gridSize-1);
}
public T set(int x, int y, T e) {
if (!inRange(x,y)) return null;
return set(_indexOf(x, y), e);
}
public T setFirst(T e) {
return set(0,0,e);
}
public T setLast(T e) {
return set(gridSize-1, gridSize-1, e);
}
public boolean inRange(int x, int y) {
return (x>=0 && x<gridSize &&