wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement cache invalidation, and have render cache be used. Can expect faster load times.

Browse Source
This commit is contained in:
tom lee
2023-08-19 16:49:09 +08:00
parent 158a614482
commit a30bfdf397
9 changed files with 44 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
+3
View File
@@ -98,6 +98,7 @@ public class ColumnRenderSource
this.verticalDataCount = parsedColumnData.verticalSize;
this.renderDataContainer = parsedColumnData.dataContainer;
this.worldGenStep = parsedColumnData.worldGenStep;
this.isEmpty = parsedColumnData.isEmpty;
this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE];
this.fillDebugFlag(0, 0, SECTION_SIZE, SECTION_SIZE, DebugSourceFlag.FILE);
@@ -263,6 +264,7 @@ public class ColumnRenderSource
this.debugSourceFlags[i / this.verticalDataCount] = renderSource.debugSourceFlags[i / this.verticalDataCount];
}
}
localVersion.incrementAndGet();
}
/**
* If the newVerticalSize is different than the current verticalSize,
@@ -275,6 +277,7 @@ public class ColumnRenderSource
{
this.verticalDataCount = newVerticalSize;
this.renderDataContainer = new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount];
localVersion.incrementAndGet();
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java
+6 -3
View File
@@ -185,9 +185,6 @@ public class FullDataFileHandler implements IFullDataSourceProvider
}
}
protected FullDataMetaFile getLoadOrMakeFile(DhSectionPos pos, boolean allowCreateFile)
{
FullDataMetaFile metaFile = this.fileBySectionPos.get(pos);
@@ -603,6 +600,12 @@ public class FullDataFileHandler implements IFullDataSourceProvider
@Override
public ExecutorService getIOExecutor() { return fileHandlerThreadPool; }
@Override
public FullDataMetaFile getFileIfExist(DhSectionPos pos)
{
return getLoadOrMakeFile(pos, false);
}
//=========//
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataMetaFile.java
+1 -1
View File
@@ -358,7 +358,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile implements I
{
AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(data.getClass(), data.getBinaryDataFormatVersion());
return new BaseMetaData(data.getSectionPos(), -1,
data.getDataDetailLevel(), data.getWorldGenStep(), (loader == null ? 0 : loader.datatypeId), data.getBinaryDataFormatVersion());
data.getDataDetailLevel(), data.getWorldGenStep(), (loader == null ? 0 : loader.datatypeId), data.getBinaryDataFormatVersion(), Long.MAX_VALUE);
}
/**
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java
+4 -1
View File
@@ -4,6 +4,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedF
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.util.Collection;
@@ -30,5 +31,7 @@ public interface IFullDataSourceProvider extends AutoCloseable
CompletableFuture<IFullDataSource> onDataFileUpdate(IFullDataSource source, FullDataMetaFile file, Consumer<IFullDataSource> onUpdated, Function<IFullDataSource, Boolean> updater);
File computeDataFilePath(DhSectionPos pos);
ExecutorService getIOExecutor();
@Nullable
FullDataMetaFile getFileIfExist(DhSectionPos pos);
}
core/src/main/java/com/seibel/distanthorizons/core/file/metaData/AbstractMetaDataContainerFile.java
+5 -6
View File
@@ -155,11 +155,11 @@ public abstract class AbstractMetaDataContainerFile
byte loaderVersion = byteBuffer.get();
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(byteBuffer.get());
long dataTypeId = byteBuffer.getLong();
long unusedTimestamp = byteBuffer.getLong(); // not currently implemented
long dataVersion = byteBuffer.getLong(); // data versioning
LodUtil.assertTrue(byteBuffer.remaining() == METADATA_RESERVED_SIZE);
DhSectionPos dataPos = new DhSectionPos(detailLevel, x, z);
return new BaseMetaData(dataPos, checksum, dataLevel, worldGenStep, dataTypeId, loaderVersion);
return new BaseMetaData(dataPos, checksum, dataLevel, worldGenStep, dataTypeId, loaderVersion, dataVersion);
}
}
@@ -214,13 +214,12 @@ public abstract class AbstractMetaDataContainerFile
try (FileChannel fileChannel = FileChannel.open(tempFile.toPath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))
{
fileChannel.position(METADATA_SIZE_IN_BYTES);
int checksum;
try (DhDataOutputStream compressedOut = new DhDataOutputStream(Channels.newOutputStream(fileChannel));
CheckedOutputStream checkedOut = new CheckedOutputStream(compressedOut, new Adler32())) // TODO: Is Adler32 ok?
{
dataWriterFunc.writeBufferToFile(compressedOut);
checksum = (int) checkedOut.getChecksum().getValue();
this.baseMetaData.checksum = (int) checkedOut.getChecksum().getValue();
}
@@ -231,13 +230,13 @@ public abstract class AbstractMetaDataContainerFile
buffer.putInt(this.pos.sectionX);
buffer.putInt(Integer.MIN_VALUE); // Unused - y pos
buffer.putInt(this.pos.sectionZ);
buffer.putInt(checksum);
buffer.putInt(this.baseMetaData.checksum);
buffer.put(this.pos.sectionDetailLevel);
buffer.put(this.baseMetaData.dataLevel);
buffer.put(this.baseMetaData.binaryDataFormatVersion);
buffer.put(this.baseMetaData.worldGenStep != null ? this.baseMetaData.worldGenStep.value : EDhApiWorldGenerationStep.EMPTY.value); // TODO this null check shouldn't be necessary
buffer.putLong(this.baseMetaData.dataTypeId);
buffer.putLong(Long.MAX_VALUE); //buff.putLong(this.metaData.dataVersion.get()); // not currently implemented
buffer.putLong(this.baseMetaData.dataVersion.get()); // for types that doesn't need data versioning, this will be Long.MAX_VALUE
LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE);
buffer.flip();
fileChannel.write(buffer);
core/src/main/java/com/seibel/distanthorizons/core/file/metaData/BaseMetaData.java
+5 -3
View File
@@ -5,6 +5,8 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.I
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import java.util.concurrent.atomic.AtomicLong;
/**
* Contains and represents the meta information ({@link DhSectionPos}, {@link BaseMetaData#dataLevel}, etc.)
* stored at the beginning of files that use the {@link AbstractMetaDataContainerFile}. <Br>
@@ -14,7 +16,7 @@ public class BaseMetaData
{
public DhSectionPos pos;
public int checksum;
// public AtomicLong dataVersion; // currently broken
public AtomicLong dataVersion = new AtomicLong(Long.MAX_VALUE);
public byte dataLevel; // TODO what does this represent?
public EDhApiWorldGenerationStep worldGenStep;
@@ -25,11 +27,11 @@ public class BaseMetaData
public BaseMetaData(DhSectionPos pos, int checksum, byte dataLevel, EDhApiWorldGenerationStep worldGenStep, long dataTypeId, byte binaryDataFormatVersion)
public BaseMetaData(DhSectionPos pos, int checksum, byte dataLevel, EDhApiWorldGenerationStep worldGenStep, long dataTypeId, byte binaryDataFormatVersion, long dataVersion)
{
this.pos = pos;
this.checksum = checksum;
// this.dataVersion = new AtomicLong(dataVersion);
this.dataVersion = new AtomicLong(dataVersion);
this.dataLevel = dataLevel;
this.worldGenStep = worldGenStep;
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderMetaDataFile.java
+2 -1
View File
@@ -223,6 +223,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements
else
{
if (!doTriggerUpdate) return CompletableFuture.completedFuture(cachedRenderDataSource);
// Make a new future, and CAS it, or return the existing future
CompletableFuture<ColumnRenderSource> newFuture = new CompletableFuture<>();
CompletableFuture<ColumnRenderSource> cas = AtomicsUtil.compareAndExchange(renderSourceLoadFutureRef, null, newFuture);
@@ -334,7 +335,7 @@ public class RenderMetaDataFile extends AbstractMetaDataContainerFile implements
private BaseMetaData makeMetaData(ColumnRenderSource renderSource)
{
return new BaseMetaData(renderSource.getSectionPos(), -1,
renderSource.getDataDetail(), renderSource.worldGenStep, RenderSourceFileHandler.RENDER_SOURCE_TYPE_ID, renderSource.getRenderDataFormatVersion());
renderSource.getDataDetail(), renderSource.worldGenStep, RenderSourceFileHandler.RENDER_SOURCE_TYPE_ID, renderSource.getRenderDataFormatVersion(), Long.MAX_VALUE);
}
private FileInputStream getFileInputStream() throws IOException
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java
+16 -1
View File
@@ -3,6 +3,7 @@ package com.seibel.distanthorizons.core.file.renderfile;
import com.google.common.collect.HashMultimap;
import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataMetaFile;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.level.ClientLevelModule;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -18,6 +19,7 @@ import com.seibel.distanthorizons.core.util.FileScanUtil;
import com.seibel.distanthorizons.core.util.FileUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.Reference;
import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException;
import com.seibel.distanthorizons.core.config.Config;
import org.apache.logging.log4j.Logger;
@@ -35,6 +37,8 @@ import static com.seibel.distanthorizons.core.util.FileScanUtil.RENDER_FILE_POST
public class RenderSourceFileHandler implements ILodRenderSourceProvider
{
public static final boolean USE_LAZY_LOADING = true;
public static final boolean ALWAYS_INVALIDATE_CACHE = false;
public static final long RENDER_SOURCE_TYPE_ID = ColumnRenderSource.TYPE_ID;
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@@ -441,12 +445,23 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
{
DebugRenderer.BoxWithLife box = new DebugRenderer.BoxWithLife(new DebugRenderer.Box(renderSource.sectionPos, 74f, 86f, 0.1f, Color.red), 1.0, 32f, Color.green.darker());
// Skip updating the cache if the data file is already up-to-date
FullDataMetaFile dataFile = this.fullDataSourceProvider.getFileIfExist(file.pos);
if (!ALWAYS_INVALIDATE_CACHE && dataFile != null && dataFile.baseMetaData.checksum == file.baseMetaData.dataVersion.get()) {
LOGGER.info("Skipping render cache update for {}", file.pos);
renderSource.localVersion.incrementAndGet();
return CompletableFuture.completedFuture(null);
}
// get the full data source loading future
final Reference<Long> targetChecksumVersion = new Reference<>(Long.MAX_VALUE);
CompletableFuture<IFullDataSource> fullDataSourceFuture =
this.fullDataSourceProvider.read(renderSource.getSectionPos())
.thenApply((fullDataSource) -> {
// the fullDataSource can be null if the thread this was running on was interrupted
box.box.color = Color.yellow.darker();
FullDataMetaFile file2 = this.fullDataSourceProvider.getFileIfExist(file.pos);
targetChecksumVersion.value = file2 == null ? Long.MAX_VALUE : file2.baseMetaData.checksum;
return fullDataSource;
}).exceptionally((ex) ->
{
@@ -468,6 +483,7 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
{
try
{
file.baseMetaData.dataVersion.set(targetChecksumVersion.value);
this.writeRenderSourceToFile(renderSource, file, newRenderSource);
}
catch (Throwable e)
@@ -522,7 +538,6 @@ public class RenderSourceFileHandler implements ILodRenderSourceProvider
}
currentRenderSource.updateFromRenderSource(newRenderSource);
currentRenderSource.localVersion.incrementAndGet();
//file.metaData.dataVersion.set(newDataVersion);
file.baseMetaData.dataLevel = currentRenderSource.getDataDetail();
core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java
+2 -2
View File
@@ -115,7 +115,7 @@ public class LodRenderSection implements IDebugRenderable
if (!this.isRenderingEnabled)
{
// this only needs to be called when first enabling the section
this.markBufferDirty();
//this.markBufferDirty();
}
this.isRenderingEnabled = true;
@@ -284,7 +284,7 @@ public class LodRenderSection implements IDebugRenderable
/** @return true if this section is loaded and set to render */
public boolean canBuildBuffer() { return this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); }
private boolean isBufferOutdated() { return this.neighborUpdated || (this.renderSource.localVersion.get() - this.lastSwapLocalVersion) > 0; }
private boolean isBufferOutdated() { return this.neighborUpdated || this.renderSource.localVersion.get() != this.lastSwapLocalVersion; }
/** @return true if this section is loaded and set to render */
public boolean canSwapBuffer() { return this.buildRenderBufferFuture != null && this.buildRenderBufferFuture.isDone(); }