wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle non-adjacent data conversion

Browse Source
This commit is contained in:
James Seibel
2025-11-06 21:28:25 -06:00
parent 9343854b4a
commit 6456651d27
4 changed files with 108 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+34 -2
View File
@@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataOcclusionCuller;
import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -36,6 +37,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos;
import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil;
import com.seibel.distanthorizons.core.util.*;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
@@ -75,8 +77,6 @@ public class FullDataSourceV2
/** how many chunks wide this datasource is at detail level 0. */
public static final int NUMB_OF_CHUNKS_WIDE = WIDTH / LodUtil.CHUNK_WIDTH;
public static final byte DATA_FORMAT_VERSION = 1;
public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("FullDataV2");
@@ -1094,6 +1094,38 @@ public class FullDataSourceV2
//===================//
// adjacent clearing //
//===================//
/** Removes any non-adjacent data from the given direction. */
public void clearAllNonAdjData(EDhDirection direction)
{
long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction);
int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos);
int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos);
int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos);
int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos);
for (int relX = 0; relX < FullDataSourceV2.WIDTH; relX++)
{
for (int relZ = 0; relZ < FullDataSourceV2.WIDTH; relZ++)
{
// skip non-adjacent data
if (relX >= minX && relX < maxX
&& relZ >= minZ && relZ < maxZ)
{
continue;
}
LongArrayList dataColumn = this.getColumnAtRelPos(relX, relZ);
dataColumn.clear();
dataColumn.add(FullDataPointUtil.EMPTY_DATA_POINT);
}
}
}
//================//
// helper methods //
//================//
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java
+31 -8
View File
@@ -19,6 +19,7 @@
package com.seibel.distanthorizons.core.file.fullDatafile.V2;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.enums.EDhDirection;
@@ -207,8 +208,19 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable
try
{
// load from database
return this.createDataSourceFromDto(dto);
FullDataSourceV2 dataSource = this.createDataSourceFromDto(dto);
// automatically create and save adjacent data if missing
if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA)
{
EDhApiDataCompressionMode compressionMode = Config.Common.LodBuilding.dataCompression.get();
try(FullDataSourceV2DTO updatedDto = FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionMode))
{
this.repo.save(updatedDto);
}
}
return dataSource;
}
catch (DataCorruptedException e)
{
@@ -262,6 +274,19 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable
return FullDataSourceV2.createEmpty(pos);
}
// migrate to the V2 format first if needed
if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA)
{
// get automatically converts from V1 to V2
FullDataSourceV2 migratedDataSource = this.get(pos);
if (migratedDataSource != null)
{
migratedDataSource.clearAllNonAdjData(direction);
}
return migratedDataSource;
}
try
{
// load from database
@@ -346,14 +371,12 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable
//
// TODO
//
//=============//
// data update //
//=============//
public CompletableFuture<Void> updateDataSourceAsync(@NotNull FullDataSourceV2 inputData)
{
return this.dataUpdater.updateDataSourceAsync(inputData);
}
{ return this.dataUpdater.updateDataSourceAsync(inputData); }
core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java
-3
View File
@@ -19,8 +19,6 @@
package com.seibel.distanthorizons.core.render;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer;
@@ -35,7 +33,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler;
import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
import com.seibel.distanthorizons.core.util.KeyedLockContainer;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.PerfRecorder;
import com.seibel.distanthorizons.core.util.ThreadUtil;
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+43 -19
View File
@@ -54,6 +54,13 @@ public class FullDataSourceV2DTO
{
public static final boolean VALIDATE_INPUT_DATAPOINTS = true;
public static class DATA_FORMAT
{
public static final int V1_NO_ADJACENT_DATA = 1;
public static final int V2_LATEST = 2;
}
public long pos;
@@ -118,7 +125,7 @@ public class FullDataSourceV2DTO
// the mapping hash isn't included since it takes significantly longer to calculate and
// as of the time of this comment (2025-1-22) the checksum isn't used for anything so changing it shouldn't cause any issues
dto.dataChecksum = dataSource.hashCode();
dto.dataFormatVersion = FullDataSourceV2.DATA_FORMAT_VERSION;
dto.dataFormatVersion = DATA_FORMAT.V2_LATEST;
dto.compressionModeValue = compressionModeEnum.value;
dto.lastModifiedUnixDateTime = dataSource.lastModifiedUnixDateTime;
dto.createdUnixDateTime = dataSource.createdUnixDateTime;
@@ -161,7 +168,7 @@ public class FullDataSourceV2DTO
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos);
try
{
this.internalPopulateDataSource(dataSource, levelWrapper, direction, false);
this.populateDataSource(dataSource, levelWrapper, direction, false);
}
catch (Exception e)
{
@@ -172,38 +179,57 @@ public class FullDataSourceV2DTO
return dataSource;
}
/**
* May be missing one or more data fields. <br>
* Designed to be used without access to Minecraft.
*/
public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException
{ return this.createUnitTestDataSource(null); }
/**
* May be missing one or more data fields. <br>
* Designed to be used without access to Minecraft or any supporting objects.
* Designed to be used without access to Minecraft.
*/
public FullDataSourceV2 createUnitTestDataSource(EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException
{ return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); }
{ return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); }
private FullDataSourceV2 internalPopulateDataSource(
private FullDataSourceV2 populateDataSource(
FullDataSourceV2 dataSource, ILevelWrapper levelWrapper,
@Nullable EDhDirection direction,
boolean unitTest) throws IOException, InterruptedException, DataCorruptedException
{
if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion)
// format validation //
if (DATA_FORMAT.V1_NO_ADJACENT_DATA != this.dataFormatVersion
&& DATA_FORMAT.V2_LATEST != this.dataFormatVersion)
{
throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"].");
throw new IllegalStateException("Data source population only supports formats: ["+DATA_FORMAT.V1_NO_ADJACENT_DATA +","+DATA_FORMAT.V2_LATEST +"], data format found: ["+this.dataFormatVersion+"].");
}
if (direction != null
&& this.dataFormatVersion == DATA_FORMAT.V1_NO_ADJACENT_DATA)
{
throw new IllegalStateException("Data format ["+this.dataFormatVersion+"] doesn't support adjacent data. Automatic conversion must be done.");
}
// compression //
EDhApiDataCompressionMode compressionModeEnum;
try
{
compressionModeEnum = this.getCompressionMode();
compressionModeEnum = EDhApiDataCompressionMode.getFromValue(this.compressionModeValue);
}
catch (IllegalArgumentException e)
{
// may happen if ZStd was used (which was added and removed during the nightly builds)
// or if the compressor value is changed to an invalid option
// may happen if the compressor value was changed to an invalid option
throw new DataCorruptedException(e);
}
// data //
if (direction == null)
{
readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum);
@@ -220,6 +246,9 @@ public class FullDataSourceV2DTO
readDataSourceAdjacentDataArrayToBlob(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum);
}
// mapping //
dataSource.mapping.clear(dataSource.getPos());
// should only be null when used in a unit test
if (!unitTest)
@@ -238,6 +267,10 @@ public class FullDataSourceV2DTO
}
}
// individual properties //
dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime;
dataSource.createdUnixDateTime = this.createdUnixDateTime;
@@ -582,15 +615,6 @@ public class FullDataSourceV2DTO
//================//
// helper methods //
//================//
public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException
{ return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); }
//===========//
// overrides //
//===========//