From 5fd8ed840f2b8b1348f11aa54da4286edf879ae9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 6 Nov 2025 07:35:23 -0600 Subject: [PATCH] Add adjacent data to FullDataDTO for faster loading --- .../methods/data/DhApiTerrainDataRepo.java | 2 +- .../render/CachedColumnRenderSource.java | 96 ------- .../FullDataToRenderDataTransformer.java | 6 +- .../GeneratedFullDataSourceProvider.java | 2 +- .../RemoteFullDataSourceProvider.java | 8 +- .../V2/FullDataSourceProviderV2.java | 120 ++++++--- .../V2/FullDataUpdatePropagatorV2.java | 8 +- .../fullDatafile/V2/FullDataUpdaterV2.java | 28 +- .../core/generation/PregenManager.java | 4 +- .../distanthorizons/core/jar/JarMain.java | 2 +- .../core/level/DhClientLevel.java | 2 +- .../AbstractFullDataNetworkRequestQueue.java | 2 +- .../server/FullDataSourceRequestHandler.java | 4 +- .../core/render/LodQuadTree.java | 86 +----- .../core/render/LodRenderSection.java | 74 ++---- .../core/sql/dto/FullDataSourceV2DTO.java | 165 +++++++++++- .../sql/dto/util/FullDataMinMaxPosUtil.java | 108 ++++++++ .../core/sql/repo/FullDataSourceV2Repo.java | 249 +++++++++++++++++- ...0090-sqlite-addAdjacentFullDataColumns.sql | 13 + .../main/resources/sqlScripts/scriptList.txt | 1 + .../java/tests/DhFullDataSourceRepoTests.java | 123 +++++++-- .../java/tests/FullDataMinMaxPosTest.java | 54 ++++ 22 files changed, 834 insertions(+), 323 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java create mode 100644 core/src/main/resources/sqlScripts/0090-sqlite-addAdjacentFullDataColumns.sql create mode 100644 core/src/test/java/tests/FullDataMinMaxPosTest.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index c27b8a58e..0dd8f2d15 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -238,7 +238,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo if (dataSource == null) { // attempt to get/generate the data source for this section - dataSource = level.getFullDataProvider().getAsync(sectionPos).get(); + dataSource = level.getFullDataProvider().getAsync(sectionPos, false).get(); if (dataSource == null) { return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + DhSectionPos.toString(sectionPos) + "]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java deleted file mode 100644 index 330186ae5..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.seibel.distanthorizons.core.dataObjects.render; - -import com.google.common.cache.Cache; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Wrapper for {@link ColumnRenderSource} that handles reference counting - * and cache tracking. - */ -public class CachedColumnRenderSource implements AutoCloseable -{ - /** an externally handled future that will complete once the {@link CachedColumnRenderSource#columnRenderSource} has finished loading */ - public final CompletableFuture loadFuture; - /** will be null initially, should be non-null once the corresponding load future is done */ - @Nullable - public ColumnRenderSource columnRenderSource = null; - - private final AtomicInteger referenceCount; - private final Cache cachedRenderSourceByPos; - private final ReentrantLock getterLock; - - - - //=============// - // constructor // - //=============// - - public CachedColumnRenderSource( - @NotNull CompletableFuture loadFuture, - @NotNull ReentrantLock getterLock, - @NotNull Cache cachedRenderSourceByPos) - { - this.loadFuture = loadFuture; - this.getterLock = getterLock; - this.referenceCount = new AtomicInteger(1); - this.cachedRenderSourceByPos = cachedRenderSourceByPos; - } - - - - //====================// - // reference counting // - //====================// - - public void markInUse() { this.referenceCount.getAndIncrement(); } - - - - //================// - // base overrides // - //================// - - /** - * Will be called multiple times, - * however it will only close the underlying data once - * all references have closed. - */ - @Override - public void close() throws IllegalStateException - { - try - { - // lock to prevent other threads for accessing the cache if we invalidate it - this.getterLock.lock(); - - // should only happen if something goes wrong up-stream - if (this.columnRenderSource == null) - { - return; - } - - - // only close once everyone is done with this datasource - int refCount = this.referenceCount.decrementAndGet(); - if (refCount == 0) - { - this.cachedRenderSourceByPos.invalidate(this.columnRenderSource.pos); - this.columnRenderSource.close(); - } - else if (refCount < 0) - { - throw new IllegalStateException("Render source ["+this.columnRenderSource.pos+"] reference count incorrect. Object already closed."); - } - } - finally - { - this.getterLock.unlock(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 5d43cea48..52fa961e0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -66,7 +66,8 @@ public class FullDataToRenderDataTransformer //==============================// @Nullable - public static ColumnRenderSource transformFullDataToRenderSource(@Nullable FullDataSourceV2 fullDataSource, @Nullable IClientLevelWrapper levelWrapper) + public static ColumnRenderSource transformFullDataToRenderSource( + @Nullable FullDataSourceV2 fullDataSource, @Nullable IClientLevelWrapper levelWrapper) { if (fullDataSource == null) { @@ -102,7 +103,8 @@ public class FullDataToRenderDataTransformer * @throws InterruptedException Can be caused by interrupting the thread upstream. * Generally thrown if the method is running after the client leaves the current world. */ - private static ColumnRenderSource transformCompleteFullDataToColumnData(IClientLevelWrapper levelWrapper, FullDataSourceV2 fullDataSource) throws InterruptedException + private static ColumnRenderSource transformCompleteFullDataToColumnData( + IClientLevelWrapper levelWrapper, FullDataSourceV2 fullDataSource) throws InterruptedException { final long pos = fullDataSource.getPos(); final byte dataDetail = fullDataSource.getDataDetailLevel(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 7d21a5898..47040c4a7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -489,7 +489,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im @Override public CompletableFuture shouldGenerateSplitChild(long pos) { - return GeneratedFullDataSourceProvider.this.getAsync(pos).thenApply(fullDataSource -> + return GeneratedFullDataSourceProvider.this.getAsync(pos, false).thenApply(fullDataSource -> { //noinspection TryFinallyCanBeTryWithResources try diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java index b3dbae741..618523718 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java @@ -75,18 +75,18 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide @Override @Nullable - public FullDataSourceV2 get(long pos) + public FullDataSourceV2 get(long pos, boolean includeAdjacentData) { if (this.syncOnLoadRequestQueue == null) { // we have local data, but networking is unavailable. - return super.get(pos); + return super.get(pos, includeAdjacentData); } if (!this.visitedPositions.add(pos)) { // This position has already been accessed before - return super.get(pos); + return super.get(pos, includeAdjacentData); } @@ -105,7 +105,7 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide }); } - return super.get(pos); + return super.get(pos, includeAdjacentData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index 56e697aeb..d3d83bf23 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.file.fullDatafile.V2; -import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc; import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; @@ -129,33 +129,6 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable - //====================// - // Abstract overrides // - //====================// - - public FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource) - { - try - { - // when creating new data use the compressor currently selected in the config - EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get(); - return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum); - } - catch (IOException e) - { - LOGGER.warn("Unable to create DTO, error: "+e.getMessage(), e); - return null; - } - } - - protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException - { return dto.createDataSource(this.level.getLevelWrapper()); } - - protected FullDataSourceV2 makeEmptyDataSource(long pos) - { return FullDataSourceV2.createEmpty(pos); } - - - //=================// // event listeners // //=================// @@ -177,6 +150,17 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable + //================// + // DTO converters // + //================// + + protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException + { return dto.createDataSource(this.level.getLevelWrapper(), null); } + protected FullDataSourceV2 createAdjDataSourceFromDto(FullDataSourceV2DTO dto, EDhDirection direction) throws InterruptedException, IOException, DataCorruptedException + { return dto.createDataSource(this.level.getLevelWrapper(), direction); } + + + //=========================// // basic DataSource getter // //=========================// @@ -187,7 +171,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable * * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ - public CompletableFuture getAsync(long pos) + public CompletableFuture getAsync(long pos, boolean includeAdjacentData) { AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) @@ -198,7 +182,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable try { - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + return CompletableFuture.supplyAsync(() -> this.get(pos, includeAdjacentData), executor); } catch (RejectedExecutionException ignore) { @@ -209,16 +193,18 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable /** * Should only be used in internal file handler methods where we are already running on a file handler thread. * Can return null if the repo is in the process of being shut down - * @see FullDataSourceProviderV2#getAsync(long) + * @see FullDataSourceProviderV2#getAsync(long, boolean) */ @Nullable - public FullDataSourceV2 get(long pos) + public FullDataSourceV2 get(long pos, boolean includeAdjacentData) { - try(FullDataSourceV2DTO dto = this.repo.getByKey(pos)) + try(FullDataSourceV2DTO dto = includeAdjacentData + ? this.repo.getByKey(pos) + : this.repo.getByPosNoAdj(pos)) { if (dto == null) { - return this.makeEmptyDataSource(pos); + return FullDataSourceV2.createEmpty(pos); } try @@ -259,6 +245,72 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable + // + // TODO name? + // + + public FullDataSourceV2 getAdjForDirection(long pos, EDhDirection direction) + { + try(FullDataSourceV2DTO dto = this.repo.getAdjByPosAndDirection(pos, direction)) + { + if (dto == null) + { + return FullDataSourceV2.createEmpty(pos); + } + + try + { + // load from database + return this.createAdjDataSourceFromDto(dto, direction); + } + catch (DataCorruptedException e) + { + this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); + this.repo.deleteWithKey(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); + } + + // an error occurred + return null; + } + + public FullDataSourceV2 getCenter(long pos) + { + try(FullDataSourceV2DTO dto = this.repo.getByPosNoAdj(pos)) + { + if (dto == null) + { + return FullDataSourceV2.createEmpty(pos); + } + + try + { + // load from database + return this.createDataSourceFromDto(dto); + } + catch (DataCorruptedException e) + { + this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); + this.repo.deleteWithKey(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); + } + + // an error occurred + return null; + } + + + //=======================// // retrieval (world gen) // //=======================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java index 8f47ac19f..f1de0c42e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java @@ -183,7 +183,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab parentLocked = true; this.dataUpdater.lockedPosSet.add(parentUpdatePos); - try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos, false)) { // will return null if the file handler is shutting down if (parentDataSource != null) @@ -197,7 +197,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab childReadLock.lock(); this.dataUpdater.lockedPosSet.add(childPos); - try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) + try (FullDataSourceV2 childDataSource = this.provider.get(childPos, false)) { // can return null when the file handler is being shut down if (childDataSource != null) @@ -299,7 +299,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab parentLocked = true; this.dataUpdater.lockedPosSet.add(parentUpdatePos); - try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos, false)) { // will return null if the file handler is shutting down if (parentDataSource != null) @@ -315,7 +315,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab childWriteLock.lock(); this.dataUpdater.lockedPosSet.add(childPos); - try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) + try (FullDataSourceV2 childDataSource = this.provider.get(childPos, false)) { // will return null if the file handler is shutting down if (childDataSource != null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java index c7c6352e9..c18a651b2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java @@ -1,5 +1,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile.V2; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc; import com.seibel.distanthorizons.core.logging.DhLogger; @@ -13,6 +15,7 @@ import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.jetbrains.annotations.NotNull; import java.awt.*; +import java.io.IOException; import java.util.ArrayList; import java.util.Set; import java.util.concurrent.*; @@ -119,7 +122,7 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable // get or create the data source - try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos)) + try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos, false)) { if (recipientDataSource != null) { @@ -127,9 +130,12 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable if (dataModified) { // save the updated data to the database - try (FullDataSourceV2DTO dto = this.provider.createDtoFromDataSource(recipientDataSource)) + try (FullDataSourceV2DTO dto = this.createDtoFromDataSource(recipientDataSource)) { - this.provider.repo.save(dto); + if (dto != null) + { + this.provider.repo.save(dto); + } } @@ -158,6 +164,22 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable } } + private FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource) + { + try + { + // when creating new data use the compressor currently selected in the config + EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get(); + return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum); + } + catch (IOException e) + { + LOGGER.warn("Unable to create DTO, error: ["+e.getMessage() + "].", e); + return null; + } + } + + //==================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java index c8f368419..53cabc7ef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java @@ -140,7 +140,9 @@ public class PregenManager } this.pendingGenerations.put(nextSectionPos, System.currentTimeMillis()); - this.fullDataSourceProvider.getAsync(nextSectionPos).thenAccept(fullDataSource -> { + this.fullDataSourceProvider.getAsync(nextSectionPos, false) + .thenAccept(fullDataSource -> + { if (this.fullDataSourceProvider.isFullyGenerated(fullDataSource.columnGenerationSteps)) { this.pendingGenerations.invalidate(fullDataSource.getPos()); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java index 806b3baeb..f3ff831d3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java @@ -239,7 +239,7 @@ public class JarMain private static void exportLodDataAtPosition(FullDataSourceV2Repo repo, File exportFile, long pos) { - FullDataSourceV2DTO dto = repo.getByKey(pos); + FullDataSourceV2DTO dto = repo.getByPosNoAdj(pos); if (dto == null) { LOGGER.error("Unable to find any data at the position ["+DhSectionPos.toString(pos)+"]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 6823db53d..fef19a6da 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -182,7 +182,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel } - FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.levelWrapper); + FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.levelWrapper, null); this.updateDataSourcesAsync(fullDataSource) .whenComplete((result, e) -> fullDataSource.close()); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java index cc8294f3c..b9608e566 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java @@ -279,7 +279,7 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende { this.level.updateBeaconBeamsForSectionPos(dataSourceDto.pos, response.payload.beaconBeams); - FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper()); + FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper(), null); entry.dataSourceConsumer.accept(fullDataSource); } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java index 2e523ea25..0b0be6c17 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java @@ -99,7 +99,7 @@ public class FullDataSourceRequestHandler } // get the server's datasource - return this.fullDataSourceProvider().get(message.sectionPos); + return this.fullDataSourceProvider().get(message.sectionPos, false); } catch (Exception e) { @@ -262,7 +262,7 @@ public class FullDataSourceRequestHandler private void tryFulfillDataSourceRequestGroup(DataSourceRequestGroup requestGroup, long pos) { - this.fullDataSourceProvider().getAsync(pos).thenAccept(fullDataSource -> + this.fullDataSourceProvider().getAsync(pos, false).thenAccept(fullDataSource -> { if (this.fullDataSourceProvider().isFullyGenerated(fullDataSource.columnGenerationSteps)) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 4b87eeeba..fb4cdde72 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -22,7 +22,6 @@ package com.seibel.distanthorizons.core.render; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.render.CachedColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer; import com.seibel.distanthorizons.core.enums.EDhDirection; @@ -87,28 +86,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen private ArrayList altDebugRenderSections = new ArrayList<>(); private final ReentrantLock debugRenderSectionLock = new ReentrantLock(); - - /** don't let two threads load the same position at the same time */ - protected final KeyedLockContainer renderLoadLockContainer = new KeyedLockContainer<>(); - - /** - * caching is done at the QuadTree level to prevent caching LODs for different levels. - * (Although the incorrect terrain that renders is quite entertaining).

- * - * caching the loaded positions significantly improves initial loading performance - * since the same position doesn't need to be loaded 5 times. - */ - private final Cache cachedRenderSourceByPos - = CacheBuilder.newBuilder() - // availableProcessors() : each process may need to be loading a render source - // +1 : add 1 thread count buffer to reduce the chance of accidentally unloading a render source before it's used - // *5 : each render source needs it's 4 adjacent sides, so a total of 5 render sources are needed per load - .maximumSize((Runtime.getRuntime().availableProcessors() + 1) * 5L) - // No closing logic since the CachedColumnRenderSource is in charge - // of freeing the underlying ColumnRenderSource. - // That way we don't have to worry about accidentally closing an in-use object. - .build(); - /** * Used to limit how many upload tasks are queued at once. * If all the upload tasks are queued at once, they will start uploading nearest @@ -240,7 +217,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen long rootPos = rootPosIterator.nextLong(); if (this.getNode(rootPos) == null) { - this.setValue(rootPos, new LodRenderSection(rootPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer)); + this.setValue(rootPos, new LodRenderSection(rootPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef)); } QuadNode rootNode = this.getNode(rootPos); @@ -281,7 +258,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen // create the node if (quadNode == null && this.isSectionPosInBounds(sectionPos)) // the position bounds should only fail when at the edge of the user's render distance { - rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer)); + rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef)); quadNode = rootNode.getNode(sectionPos); } if (quadNode == null) @@ -294,7 +271,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen LodRenderSection renderSection = quadNode.value; if (renderSection == null) { - renderSection = new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer); + renderSection = new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef); quadNode.setValue(sectionPos, renderSection); } @@ -636,17 +613,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen * This should be called whenever a world generation task is completed or if the connected server has new data to show. */ public void reloadPos(long pos) - { - // clear cache // - - this.clearRenderCacheForPos(pos); - for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS) - { - long adjacentPos = DhSectionPos.getAdjacentPos(pos, direction); - this.clearRenderCacheForPos(adjacentPos); - } - - + { // queue reloads // // only queue each section for reloading @@ -664,22 +631,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen this.sectionsToReload.add(adjacentPos); } } - private void clearRenderCacheForPos(long pos) - { - // locking is needed to prevent another thread - // from accessing the cache while it's being cleared - ReentrantLock lock = this.renderLoadLockContainer.getLockForPos(pos); - try - { - lock.lock(); - this.cachedRenderSourceByPos.invalidate(pos); - } - finally - { - lock.unlock(); - } - } - //=================================// @@ -830,39 +781,10 @@ public class LodQuadTree extends QuadTree implements IDebugRen LodRenderSection renderSection = quadNode.value; if (renderSection != null) { - // we need to wait for the render data to finish building before we can close the cache - CompletableFuture future = renderSection.getRenderDataBuildFuture(); - if (future != null) - { - renderDataBuildFutures.add(future); - } - renderSection.close(); quadNode.value = null; } } - - - // close the render cache after it is done being used - LOGGER.info("waiting for ["+renderDataBuildFutures.size()+"] futures before closing render cache..."); - CompletableFuture.allOf(renderDataBuildFutures.toArray(new CompletableFuture[0])) - .handle((voidObj, throwable) -> - { - // run on a separate thread so we don't lock up the main cleanup thread - // with the sleep() call - new Thread(() -> - { - // Sleep shouldn't be necessary, but James found a few cases where - // the futures incorrectly claimed they were done. - // Sleeping solved those issues. - try { Thread.sleep(5_000); } catch (InterruptedException ignore) { } - - LOGGER.debug("closing render cache"); - this.cachedRenderSourceByPos.invalidateAll(); - }).start(); - - return null; - }); } finally { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 49c52f870..9732596df 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -21,10 +21,8 @@ package com.seibel.distanthorizons.core.render; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; -import com.google.common.cache.Cache; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dataObjects.render.CachedColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodQuadBuilder; @@ -44,7 +42,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler; import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO; import com.seibel.distanthorizons.core.sql.repo.BeaconBeamRepo; -import com.seibel.distanthorizons.core.util.KeyedLockContainer; import com.seibel.distanthorizons.core.util.PerfRecorder; import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; @@ -59,7 +56,6 @@ import java.util.*; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; /** * A render section represents an area that could be rendered. @@ -79,8 +75,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable @WillNotClose private final FullDataSourceProviderV2 fullDataSourceProvider; private final LodQuadTree quadTree; - private final KeyedLockContainer renderLoadLockContainer; - private final Cache cachedRenderSourceByPos; private final AtomicInteger uploadTaskCountRef; /** @@ -147,13 +141,10 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable long pos, LodQuadTree quadTree, IDhClientLevel level, FullDataSourceProviderV2 fullDataSourceProvider, - AtomicInteger uploadTaskCountRef, - Cache cachedRenderSourceByPos, KeyedLockContainer renderLoadLockContainer) + AtomicInteger uploadTaskCountRef) { this.pos = pos; this.quadTree = quadTree; - this.cachedRenderSourceByPos = cachedRenderSourceByPos; - this.renderLoadLockContainer = renderLoadLockContainer; this.level = level; this.levelWrapper = level.getClientLevelWrapper(); this.fullDataSourceProvider = fullDataSourceProvider; @@ -245,11 +236,11 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { // get the center pos data return this.getRenderSourceForPosAsync(this.pos, null) - .thenCompose((CachedColumnRenderSource cachedRenderSource) -> + .thenCompose((ColumnRenderSource thisRenderSource) -> { try { - if (cachedRenderSource == null || cachedRenderSource.columnRenderSource == null) + if (thisRenderSource == null) { // nothing needs to be rendered // TODO how doesn't this cause infinite file handler loops? @@ -257,7 +248,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // setting the render buffer here return CompletableFuture.completedFuture(null); } - ColumnRenderSource thisRenderSource = cachedRenderSource.columnRenderSource; boolean enableTransparency = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled; @@ -268,7 +258,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // get the adjacent positions // needs to be done async to prevent threads waiting on the same positions to be processed - final CompletableFuture[] adjacentLoadFutures = new CompletableFuture[4]; + final CompletableFuture[] adjacentLoadFutures = new CompletableFuture[4]; if (Config.Client.Advanced.Graphics.Experimental.onlyLoadCenterLods.get()) { @@ -291,16 +281,16 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { getAdj.end(); - try (CachedColumnRenderSource northRenderSource = adjacentLoadFutures[0].get(); - CachedColumnRenderSource southRenderSource = adjacentLoadFutures[1].get(); - CachedColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get(); - CachedColumnRenderSource westRenderSource = adjacentLoadFutures[3].get()) + try (ColumnRenderSource northRenderSource = adjacentLoadFutures[0].get(); + ColumnRenderSource southRenderSource = adjacentLoadFutures[1].get(); + ColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get(); + ColumnRenderSource westRenderSource = adjacentLoadFutures[3].get()) { ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.CARDINAL_COMPASS.length]; - adjacentRenderSections[EDhDirection.NORTH.ordinal() - 2] = (northRenderSource != null) ? northRenderSource.columnRenderSource : null; - adjacentRenderSections[EDhDirection.SOUTH.ordinal() - 2] = (southRenderSource != null) ? southRenderSource.columnRenderSource : null; - adjacentRenderSections[EDhDirection.EAST.ordinal() - 2] = (eastRenderSource != null) ? eastRenderSource.columnRenderSource : null; - adjacentRenderSections[EDhDirection.WEST.ordinal() - 2] = (westRenderSource != null) ? westRenderSource.columnRenderSource : null; + adjacentRenderSections[EDhDirection.NORTH.ordinal() - 2] = northRenderSource; + adjacentRenderSections[EDhDirection.SOUTH.ordinal() - 2] = southRenderSource; + adjacentRenderSections[EDhDirection.EAST.ordinal() - 2] = eastRenderSource; + adjacentRenderSections[EDhDirection.WEST.ordinal() - 2] = westRenderSource; boolean[] adjIsSameDetailLevel = new boolean[EDhDirection.CARDINAL_COMPASS.length]; adjIsSameDetailLevel[EDhDirection.NORTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.NORTH); @@ -325,7 +315,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable finally { // can only be closed after the data has been processed and uploaded to the GPU - cachedRenderSource.close(); + thisRenderSource.close(); } }); } @@ -337,7 +327,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable }); } /** async is done so each thread can run without waiting on others */ - private CompletableFuture getRenderSourceForPosAsync(long pos, @Nullable EDhDirection direction) + private CompletableFuture getRenderSourceForPosAsync(long pos, @Nullable EDhDirection direction) { if (direction != null) { @@ -346,23 +336,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable final long finalPos = pos; - ReentrantLock lock = this.renderLoadLockContainer.getLockForPos(finalPos); try { - // we don't want multiple threads attempting to load the same position at the same time, - // and we don't want to access the cache while invalidating it on a different thread - lock.lock(); - - // use the cached data if possible - CachedColumnRenderSource existingCachedRenderSource = this.cachedRenderSourceByPos.getIfPresent(finalPos); - if (existingCachedRenderSource != null) - { - existingCachedRenderSource.markInUse(); - return existingCachedRenderSource.loadFuture; - } - - - PriorityTaskPicker.Executor executor = ThreadPoolUtil.getRenderLoadingExecutor(); if (executor == null || executor.isTerminated()) { @@ -372,31 +347,30 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // queue loading the render data - CompletableFuture loadFuture = new CompletableFuture<>(); - final CachedColumnRenderSource newCachedRenderSource = new CachedColumnRenderSource(loadFuture, lock, this.cachedRenderSourceByPos); + CompletableFuture loadFuture = new CompletableFuture<>(); executor.execute(() -> { PerfRecorder.Timer getFull = this.filePerfRecorder.start("getFull"); // generate new render source - try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(finalPos)) + try (FullDataSourceV2 fullDataSource = + // no direction means get the center LOD + (direction == null) + ? this.fullDataSourceProvider.getCenter(finalPos) + : this.fullDataSourceProvider.getAdjForDirection(finalPos, direction.opposite())) { getFull.end(); PerfRecorder.Timer transform = this.filePerfRecorder.start("transform"); - newCachedRenderSource.columnRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.levelWrapper); + ColumnRenderSource columnRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.levelWrapper); + loadFuture.complete(columnRenderSource); transform.end(); } catch (Exception e) { LOGGER.error("Unexpected issue creating render data for pos: ["+DhSectionPos.toString(finalPos)+"], error: ["+e.getMessage()+"].", e); } - finally - { - loadFuture.complete(newCachedRenderSource); - } }); - this.cachedRenderSourceByPos.put(pos, newCachedRenderSource); return loadFuture; } @@ -410,10 +384,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable LOGGER.error("Unexpected issue getting and creating render data for pos: ["+DhSectionPos.toString(pos)+"], error: ["+e.getMessage()+"].", e); return CompletableFuture.completedFuture(null); } - finally - { - lock.unlock(); - } } private boolean isAdjacentPosSameDetailLevel(EDhDirection direction) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index d49d40ea9..90b79d6c5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -25,10 +25,12 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.network.INetworkObject; +import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil; import com.seibel.distanthorizons.core.util.BoolUtil; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.ListUtil; @@ -61,6 +63,10 @@ public class FullDataSourceV2DTO public int dataChecksum; public ByteArrayList compressedDataByteArray; + public ByteArrayList compressedNorthAdjDataByteArray; + public ByteArrayList compressedSouthAdjDataByteArray; + public ByteArrayList compressedEastAdjDataByteArray; + public ByteArrayList compressedWestAdjDataByteArray; /** @see EDhApiWorldGenerationStep */ public ByteArrayList compressedColumnGenStepByteArray; @@ -100,6 +106,11 @@ public class FullDataSourceV2DTO writeGenerationStepsToBlob(dataSource.columnGenerationSteps, dto.compressedColumnGenStepByteArray, compressionModeEnum); writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, dto.compressedWorldCompressionModeByteArray, compressionModeEnum); writeDataMappingToBlob(dataSource.mapping, dto.compressedMappingByteArray, compressionModeEnum); + // adjacent full data + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedNorthAdjDataByteArray, EDhDirection.NORTH, compressionModeEnum); + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedSouthAdjDataByteArray, EDhDirection.SOUTH, compressionModeEnum); + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedEastAdjDataByteArray, EDhDirection.EAST, compressionModeEnum); + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedWestAdjDataByteArray, EDhDirection.WEST, compressionModeEnum); // populate individual variables { @@ -124,7 +135,7 @@ public class FullDataSourceV2DTO private FullDataSourceV2DTO() { - super(ARRAY_LIST_POOL, 4, 0, 0); + super(ARRAY_LIST_POOL, 8, 0, 0); // Expected sizes here are 0 since we don't know how big these arrays need to be, // they depend on compression settings and world complexity. @@ -132,6 +143,11 @@ public class FullDataSourceV2DTO this.compressedColumnGenStepByteArray = this.pooledArraysCheckout.getByteArray(1, 0); this.compressedWorldCompressionModeByteArray = this.pooledArraysCheckout.getByteArray(2, 0); this.compressedMappingByteArray = this.pooledArraysCheckout.getByteArray(3, 0); + + this.compressedNorthAdjDataByteArray = this.pooledArraysCheckout.getByteArray(4, 0); + this.compressedSouthAdjDataByteArray = this.pooledArraysCheckout.getByteArray(5, 0); + this.compressedEastAdjDataByteArray = this.pooledArraysCheckout.getByteArray(6, 0); + this.compressedWestAdjDataByteArray = this.pooledArraysCheckout.getByteArray(7, 0); } @@ -140,12 +156,12 @@ public class FullDataSourceV2DTO // data source population // //========================// - public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException + public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper, EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException { FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos); try { - this.internalPopulateDataSource(dataSource, levelWrapper, false); + this.internalPopulateDataSource(dataSource, levelWrapper, direction, false); } catch (Exception e) { @@ -156,14 +172,19 @@ public class FullDataSourceV2DTO return dataSource; } + public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException + { return this.createUnitTestDataSource(null); } /** * May be missing one or more data fields.
* Designed to be used without access to Minecraft or any supporting objects. */ - public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException - { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, true); } + public FullDataSourceV2 createUnitTestDataSource(EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException + { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); } - private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException + private FullDataSourceV2 internalPopulateDataSource( + FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, + @Nullable EDhDirection direction, + boolean unitTest) throws IOException, InterruptedException, DataCorruptedException { if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) { @@ -183,10 +204,21 @@ public class FullDataSourceV2DTO throw new DataCorruptedException(e); } - - readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); - readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum); - readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + if (direction == null) + { + readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); + readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum); + readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + } + else + { + // adjacent data is stored in the same byte array + // as the normal data, + // this is done so data sources down-stream + // can all be handled identically regardless of + // whether they're a full or partial data source + readDataSourceAdjacentDataArrayToBlob(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum); + } dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test @@ -231,7 +263,111 @@ public class FullDataSourceV2DTO // (de)serializing // //=================// - private static void writeDataSourceDataArrayToBlob(LongArrayList[] inputDataArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static void writeDataSourceAdjacentDataArrayToBlob( + LongArrayList[] wholeInputDataArray, ByteArrayList outputByteArray, + EDhDirection direction, + EDhApiDataCompressionMode compressionModeEnum) throws IOException + { + // write the outputs to a stream to prep for writing to the database + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + + // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, + // but since this stream will be closed immediately after writing anyway, it won't be an issue + try (DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum)) + { + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + + for (int x = minX; x < maxX; x++) + { + for (int z = minZ; z < maxZ; z++) + { + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList dataColumn = wholeInputDataArray[index]; + + // write column length + short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; + // a short is used instead of an int because at most we store 4096 vertical slices and a + // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) + compressedOut.writeShort(columnLength); + + // write column data (will be skipped if no data was present) + for (int y = 0; y < columnLength; y++) + { + compressedOut.writeLong(dataColumn.getLong(y)); + } + } + } + + + // generate the checksum (currently unused) + compressedOut.flush(); + byteArrayOutputStream.close(); + outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); + } + } + private static void readDataSourceAdjacentDataArrayToBlob( + @NotNull ByteArrayList inputCompressedDataByteArray, @NotNull LongArrayList[] outputDataLongArray, + @NotNull EDhDirection direction, + EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements()); + try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum)) + { + for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++) + { + @NotNull LongArrayList array = outputDataLongArray[i]; + array.clear(); + array.add(FullDataPointUtil.EMPTY_DATA_POINT); + } + + + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + + for (int x = minX; x < maxX; x++) + { + for (int z = minZ; z < maxZ; z++) + { + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList dataColumn = outputDataLongArray[index]; + + // read the column length + short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later + if (dataColumnLength < 0) + { + throw new DataCorruptedException("Read DataSource adj[" + direction + "] Blob data at index [" + index + "], column length [" + dataColumnLength + "] should be greater than zero."); + } + + + ListUtil.clearAndSetSize(dataColumn, dataColumnLength); + + // read column data (will be skipped if no data was present) + for (int y = 0; y < dataColumnLength; y++) + { + long dataPoint = compressedIn.readLong(); + if (VALIDATE_INPUT_DATAPOINTS) + { + FullDataPointUtil.validateDatapoint(dataPoint); + } + dataColumn.set(y, dataPoint); + } + } + } + + } + } + + + private static void writeDataSourceDataArrayToBlob( + LongArrayList[] inputDataArray, ByteArrayList outputByteArray, + EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -266,7 +402,9 @@ public class FullDataSourceV2DTO outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); } } - private static void readBlobToDataSourceDataArray(ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException + private static void readBlobToDataSourceDataArray( + ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, + EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements()); try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum)) @@ -448,7 +586,8 @@ public class FullDataSourceV2DTO // helper methods // //================// - public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } + public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException + { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java new file mode 100644 index 000000000..cc74e7805 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java @@ -0,0 +1,108 @@ +package com.seibel.distanthorizons.core.sql.dto.util; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; + +/** + * Handles encoding/decoding of min/max X/Z relative {@link FullDataSourceV2#dataPoints} + * positions.
+ * Needed so we can keep the same format between complete data sources + * and incomplete adjacent-only data sources. + */ +public class FullDataMinMaxPosUtil +{ + private static final int ADJ_POS_MASK = (int) Math.pow(2, Short.SIZE) - 1; + private static final int MIN_X_OFFSET = 0; + private static final int MAX_X_OFFSET = Short.SIZE; + private static final int MIN_Z_OFFSET = Short.SIZE * 2; + private static final int MAX_Z_OFFSET = Short.SIZE * 3; + + + + /** + * Encodes min/max X/Z relative {@link FullDataSourceV2#dataPoints} + * positions.
+ * Needed so we can keep the same format between complete data sources + * and incomplete adjacent-only data sources. + */ + public static long getEncodedMinMaxPos(EDhDirection direction) + { + // 4 shorts can fit in a long, and we won't need anything longer than 64 anyway + short minX; + short maxX; + short minZ; + short maxZ; + + switch (direction) + { + case NORTH: + // one row closest to the negative Z axis + minX = 0; + maxX = FullDataSourceV2.WIDTH; + + minZ = 0; + maxZ = 1; + break; + + case SOUTH: + // one row closest to the positive Z axis + minX = 0; + maxX = FullDataSourceV2.WIDTH; + + minZ = FullDataSourceV2.WIDTH - 1; + maxZ = FullDataSourceV2.WIDTH; + break; + + case EAST: + // one row closest to the positive X axis + minX = FullDataSourceV2.WIDTH - 1; + maxX = FullDataSourceV2.WIDTH; + + minZ = 0; + maxZ = FullDataSourceV2.WIDTH; + break; + + case WEST: + // one row closest to the Negative X axis + minX = 0; + maxX = 1; + + minZ = 0; + maxZ = FullDataSourceV2.WIDTH; + break; + + default: + throw new IllegalArgumentException("Unsupported direction [" + direction + "]."); + } + + return encodeAdjMinMaxPos( + minX, maxX, + minZ, maxZ); + } + + public static long encodeAdjMinMaxPos( + short minX, short maxX, + short minZ, short maxZ + ) + { + long data = 0L; + data |= (long) minX << MIN_X_OFFSET; + data |= (long) maxX << MAX_X_OFFSET; + data |= (long) minZ << MIN_Z_OFFSET; + data |= (long) maxZ << MAX_Z_OFFSET; + return data; + } + + public static int getAdjMinX(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MIN_X_OFFSET) & ADJ_POS_MASK); } + public static int getAdjMaxX(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MAX_X_OFFSET) & ADJ_POS_MASK); } + + public static int getAdjMinZ(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MIN_Z_OFFSET) & ADJ_POS_MASK); } + public static int getAdjMaxZ(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MAX_Z_OFFSET) & ADJ_POS_MASK); } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 6d9f47c5d..7ba0eeaa7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -31,7 +32,6 @@ import com.seibel.distanthorizons.core.util.ListUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; -import com.seibel.distanthorizons.core.logging.DhLogger; import org.jetbrains.annotations.Nullable; import java.io.*; @@ -84,6 +84,9 @@ public class FullDataSourceV2Repo extends AbstractDhRepo= minX && x < maxX + && z >= minZ && z < maxZ) + { + continue; + } + + + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList adjDataColumn = adjSource.dataPoints[index]; + Assert.assertEquals(1, adjDataColumn.size()); + Assert.assertEquals(FullDataPointUtil.EMPTY_DATA_POINT, adjDataColumn.getLong(0)); + } + } + } } diff --git a/core/src/test/java/tests/FullDataMinMaxPosTest.java b/core/src/test/java/tests/FullDataMinMaxPosTest.java new file mode 100644 index 000000000..5a4873f02 --- /dev/null +++ b/core/src/test/java/tests/FullDataMinMaxPosTest.java @@ -0,0 +1,54 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil; +import org.junit.Assert; +import org.junit.Test; + +public class FullDataMinMaxPosTest +{ + + @Test + public void EncodeAdjacentMinMaxPosTest() + { + int maxTest = 3; + for (short minX = 0; minX < maxTest; minX++) + { + for (short maxX = 0; maxX < maxTest; maxX++) + { + for (short minZ = 0; minZ < maxTest; minZ++) + { + for (short maxZ = 0; maxZ < maxTest; maxZ++) + { + long encodedPos = FullDataMinMaxPosUtil.encodeAdjMinMaxPos(minX, maxX, minZ, maxZ); + + Assert.assertEquals(minX, FullDataMinMaxPosUtil.getAdjMinX(encodedPos)); + Assert.assertEquals(maxX, FullDataMinMaxPosUtil.getAdjMaxX(encodedPos)); + Assert.assertEquals(minZ, FullDataMinMaxPosUtil.getAdjMinZ(encodedPos)); + Assert.assertEquals(maxZ, FullDataMinMaxPosUtil.getAdjMaxZ(encodedPos)); + } + } + } + } + + } + +}