From 6456651d2795212aef52939bc28149a23895759c Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 6 Nov 2025 21:28:25 -0600 Subject: [PATCH] Handle non-adjacent data conversion --- .../fullData/sources/FullDataSourceV2.java | 36 ++++++++++- .../V2/FullDataSourceProviderV2.java | 39 +++++++++--- .../core/render/LodQuadTree.java | 3 - .../core/sql/dto/FullDataSourceV2DTO.java | 62 +++++++++++++------ 4 files changed, 108 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index eebb24e92..71f44b254 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataOcclusionCuller; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -36,6 +37,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos; +import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil; import com.seibel.distanthorizons.core.util.*; import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; @@ -75,8 +77,6 @@ public class FullDataSourceV2 /** how many chunks wide this datasource is at detail level 0. */ public static final int NUMB_OF_CHUNKS_WIDE = WIDTH / LodUtil.CHUNK_WIDTH; - public static final byte DATA_FORMAT_VERSION = 1; - public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("FullDataV2"); @@ -1094,6 +1094,38 @@ public class FullDataSourceV2 + //===================// + // adjacent clearing // + //===================// + + /** Removes any non-adjacent data from the given direction. */ + public void clearAllNonAdjData(EDhDirection direction) + { + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + + for (int relX = 0; relX < FullDataSourceV2.WIDTH; relX++) + { + for (int relZ = 0; relZ < FullDataSourceV2.WIDTH; relZ++) + { + // skip non-adjacent data + if (relX >= minX && relX < maxX + && relZ >= minZ && relZ < maxZ) + { + continue; + } + + LongArrayList dataColumn = this.getColumnAtRelPos(relX, relZ); + dataColumn.clear(); + dataColumn.add(FullDataPointUtil.EMPTY_DATA_POINT); + } + } + } + + //================// // helper methods // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index 4487803b4..15ff70549 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile.V2; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.enums.EDhDirection; @@ -207,8 +208,19 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable try { - // load from database - return this.createDataSourceFromDto(dto); + FullDataSourceV2 dataSource = this.createDataSourceFromDto(dto); + + // automatically create and save adjacent data if missing + if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA) + { + EDhApiDataCompressionMode compressionMode = Config.Common.LodBuilding.dataCompression.get(); + try(FullDataSourceV2DTO updatedDto = FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionMode)) + { + this.repo.save(updatedDto); + } + } + + return dataSource; } catch (DataCorruptedException e) { @@ -262,6 +274,19 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable return FullDataSourceV2.createEmpty(pos); } + // migrate to the V2 format first if needed + if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA) + { + // get automatically converts from V1 to V2 + FullDataSourceV2 migratedDataSource = this.get(pos); + if (migratedDataSource != null) + { + migratedDataSource.clearAllNonAdjData(direction); + } + + return migratedDataSource; + } + try { // load from database @@ -346,14 +371,12 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable - // - // TODO - // + //=============// + // data update // + //=============// public CompletableFuture updateDataSourceAsync(@NotNull FullDataSourceV2 inputData) - { - return this.dataUpdater.updateDataSourceAsync(inputData); - } + { return this.dataUpdater.updateDataSourceAsync(inputData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index fb4cdde72..3fa3dd9e3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -19,8 +19,6 @@ package com.seibel.distanthorizons.core.render; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer; @@ -35,7 +33,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler; import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer; -import com.seibel.distanthorizons.core.util.KeyedLockContainer; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.PerfRecorder; import com.seibel.distanthorizons.core.util.ThreadUtil; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 90b79d6c5..2d7917a1b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -54,6 +54,13 @@ public class FullDataSourceV2DTO { public static final boolean VALIDATE_INPUT_DATAPOINTS = true; + public static class DATA_FORMAT + { + public static final int V1_NO_ADJACENT_DATA = 1; + public static final int V2_LATEST = 2; + } + + public long pos; @@ -118,7 +125,7 @@ public class FullDataSourceV2DTO // the mapping hash isn't included since it takes significantly longer to calculate and // as of the time of this comment (2025-1-22) the checksum isn't used for anything so changing it shouldn't cause any issues dto.dataChecksum = dataSource.hashCode(); - dto.dataFormatVersion = FullDataSourceV2.DATA_FORMAT_VERSION; + dto.dataFormatVersion = DATA_FORMAT.V2_LATEST; dto.compressionModeValue = compressionModeEnum.value; dto.lastModifiedUnixDateTime = dataSource.lastModifiedUnixDateTime; dto.createdUnixDateTime = dataSource.createdUnixDateTime; @@ -161,7 +168,7 @@ public class FullDataSourceV2DTO FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos); try { - this.internalPopulateDataSource(dataSource, levelWrapper, direction, false); + this.populateDataSource(dataSource, levelWrapper, direction, false); } catch (Exception e) { @@ -172,38 +179,57 @@ public class FullDataSourceV2DTO return dataSource; } + /** + * May be missing one or more data fields.
+ * Designed to be used without access to Minecraft. + */ public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException { return this.createUnitTestDataSource(null); } /** * May be missing one or more data fields.
- * Designed to be used without access to Minecraft or any supporting objects. + * Designed to be used without access to Minecraft. */ public FullDataSourceV2 createUnitTestDataSource(EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException - { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); } + { return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); } - private FullDataSourceV2 internalPopulateDataSource( + private FullDataSourceV2 populateDataSource( FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, @Nullable EDhDirection direction, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException { - if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) + // format validation // + + if (DATA_FORMAT.V1_NO_ADJACENT_DATA != this.dataFormatVersion + && DATA_FORMAT.V2_LATEST != this.dataFormatVersion) { - throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"]."); + throw new IllegalStateException("Data source population only supports formats: ["+DATA_FORMAT.V1_NO_ADJACENT_DATA +","+DATA_FORMAT.V2_LATEST +"], data format found: ["+this.dataFormatVersion+"]."); } + if (direction != null + && this.dataFormatVersion == DATA_FORMAT.V1_NO_ADJACENT_DATA) + { + throw new IllegalStateException("Data format ["+this.dataFormatVersion+"] doesn't support adjacent data. Automatic conversion must be done."); + } + + + + // compression // EDhApiDataCompressionMode compressionModeEnum; try { - compressionModeEnum = this.getCompressionMode(); + compressionModeEnum = EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } catch (IllegalArgumentException e) { - // may happen if ZStd was used (which was added and removed during the nightly builds) - // or if the compressor value is changed to an invalid option + // may happen if the compressor value was changed to an invalid option throw new DataCorruptedException(e); } + + + // data // + if (direction == null) { readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); @@ -220,6 +246,9 @@ public class FullDataSourceV2DTO readDataSourceAdjacentDataArrayToBlob(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum); } + + // mapping // + dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test if (!unitTest) @@ -238,6 +267,10 @@ public class FullDataSourceV2DTO } } + + + // individual properties // + dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; dataSource.createdUnixDateTime = this.createdUnixDateTime; @@ -582,15 +615,6 @@ public class FullDataSourceV2DTO - //================// - // helper methods // - //================// - - public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException - { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } - - - //===========// // overrides // //===========//