diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index e74d7bedd..0684e4cc3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -26,10 +26,7 @@ import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.world.DhApiWorldProxy; import com.seibel.distanthorizons.core.api.external.methods.config.DhApiConfig; import com.seibel.distanthorizons.core.api.external.methods.data.DhApiTerrainDataRepo; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.CompleteFullDataSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.HighDetailIncompleteFullDataSourceLoader; import com.seibel.distanthorizons.api.DhApi; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.LowDetailIncompleteFullDataSourceLoader; import com.seibel.distanthorizons.core.render.DhApiRenderProxy; //import io.netty.buffer.ByteBuf; import net.jpountz.lz4.LZ4Compressor; @@ -77,10 +74,6 @@ public class Initializer - CompleteFullDataSourceLoader unused2 = new CompleteFullDataSourceLoader(); // Auto register into the loader system - HighDetailIncompleteFullDataSourceLoader unused3 = new HighDetailIncompleteFullDataSourceLoader(); // Auto register - LowDetailIncompleteFullDataSourceLoader unused4 = new LowDetailIncompleteFullDataSourceLoader(); // Auto register - // link Core's config to the API DhApi.Delayed.configs = DhApiConfig.INSTANCE; DhApi.Delayed.terrainRepo = DhApiTerrainDataRepo.INSTANCE; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index e2d2c8d66..0cc87f208 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -28,7 +28,7 @@ import com.seibel.distanthorizons.api.objects.math.DhApiVec3i; import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhLodPos; @@ -212,7 +212,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo try { // attempt to get/generate the data source for this section - IFullDataSource dataSource = level.getFileHandler().getAsync(sectionPos).get(); + NewFullDataSource dataSource = level.getFullDataProvider().getAsync(sectionPos).get(); if (dataSource == null) { return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + sectionPos + "]."); @@ -221,7 +221,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo { // attempt to get the LOD data from the data source FullDataPointIdMap mapping = dataSource.getMapping(); - SingleColumnFullDataAccessor dataColumn = dataSource.tryGet(relativePos.x, relativePos.z); + SingleColumnFullDataAccessor dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { int dataColumnIndexCount = dataColumn.getSingleLength(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java deleted file mode 100644 index ac8862668..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData; - -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.LodUtil; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.concurrent.CompletableFuture; - -public class FullDataDownSampler -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - public static CompletableFuture createDownSamplingFuture(DhSectionPos newTarget, IFullDataSourceProvider provider) - { - // TODO: Make this future somehow run with lowest priority (to ensure ram usage stays low) - return createDownSamplingFuture(CompleteFullDataSource.createEmpty(newTarget), provider); - } - - public static CompletableFuture createDownSamplingFuture(CompleteFullDataSource target, IFullDataSourceProvider provider) - { - int sectionSizeNeeded = 1 << target.getDataDetailLevel(); - - ArrayList> futures; - DhLodPos basePos = target.getSectionPos().getSectionBBoxPos().getCornerLodPos(CompleteFullDataSource.SECTION_SIZE_OFFSET); - - - if (sectionSizeNeeded <= CompleteFullDataSource.SECTION_SIZE_OFFSET) - { - futures = new ArrayList<>(sectionSizeNeeded * sectionSizeNeeded); - for (int xOffset = 0; xOffset < sectionSizeNeeded; xOffset++) - { - for (int zOffset = 0; zOffset < sectionSizeNeeded; zOffset++) - { - CompletableFuture future = provider.getAsync(new DhSectionPos( - CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset, basePos.z + zOffset)); - future = future.whenComplete((source, ex) -> { - if (ex == null && source != null && source instanceof CompleteFullDataSource) - { - downSample(target, (CompleteFullDataSource) source); - } - else if (ex != null) - { - LOGGER.error("Error while down sampling", ex); - } - }); - futures.add(future); - } - } - } - else - { - futures = new ArrayList<>(CompleteFullDataSource.WIDTH * CompleteFullDataSource.WIDTH); - int multiplier = sectionSizeNeeded / CompleteFullDataSource.WIDTH; - for (int xOffset = 0; xOffset < CompleteFullDataSource.WIDTH; xOffset++) - { - for (int zOffset = 0; zOffset < CompleteFullDataSource.WIDTH; zOffset++) - { - CompletableFuture future = provider.getAsync(new DhSectionPos( - CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset * multiplier, basePos.z + zOffset * multiplier)); - future = future.whenComplete((source, ex) -> { - if (ex == null && source != null && source instanceof CompleteFullDataSource) - { - downSample(target, (CompleteFullDataSource) source); - } - else if (ex != null) - { - LOGGER.error("Error while down sampling", ex); - } - }); - futures.add(future); - } - } - } - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenApply(v -> target); - } - - public static void downSample(CompleteFullDataSource target, CompleteFullDataSource source) - { - LodUtil.assertTrue(target.getSectionPos().overlapsExactly(source.getSectionPos())); - LodUtil.assertTrue(target.getDataDetailLevel() > source.getDataDetailLevel()); - - byte detailDiff = (byte) (target.getDataDetailLevel() - source.getDataDetailLevel()); - DhSectionPos trgPos = target.getSectionPos(); - DhSectionPos srcPos = source.getSectionPos(); - - if (detailDiff >= CompleteFullDataSource.SECTION_SIZE_OFFSET) - { - // The source occupies only 1 datapoint in the target - // FIXME: TEMP method for down-sampling: take only the corner column - int sourceSectionPerTargetData = 1 << (detailDiff - CompleteFullDataSource.SECTION_SIZE_OFFSET); - if (srcPos.getX() % sourceSectionPerTargetData != 0 || srcPos.getZ() % sourceSectionPerTargetData != 0) - { - return; - } - DhLodPos trgOffset = trgPos.getMinCornerLodPos(target.getDataDetailLevel()); - DhLodPos srcOffset = srcPos.getSectionBBoxPos().convertToDetailLevel(target.getDataDetailLevel()); - int offsetX = trgOffset.x - srcOffset.x; - int offsetZ = trgOffset.z - srcOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < CompleteFullDataSource.WIDTH - && offsetZ >= 0 && offsetZ < CompleteFullDataSource.WIDTH); - target.markNotEmpty(); - source.get(0, 0).deepCopyTo(target.get(offsetX, offsetZ)); - - } - else if (detailDiff > 0) - { - // The source occupies multiple data-points in the target - int srcDataPerTrgData = 1 << detailDiff; - int overlappedTrgDataSize = CompleteFullDataSource.WIDTH / srcDataPerTrgData; - - DhLodPos trgOffset = trgPos.getMinCornerLodPos(target.getDataDetailLevel()); - DhLodPos srcOffset = srcPos.getSectionBBoxPos().getCornerLodPos(target.getDataDetailLevel()); - int offsetX = trgOffset.x - srcOffset.x; - int offsetZ = trgOffset.z - srcOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < CompleteFullDataSource.WIDTH - && offsetZ >= 0 && offsetZ < CompleteFullDataSource.WIDTH); - target.markNotEmpty(); - - for (int ox = 0; ox < overlappedTrgDataSize; ox++) - { - for (int oz = 0; oz < overlappedTrgDataSize; oz++) - { - SingleColumnFullDataAccessor column = target.get(ox + offsetX, oz + offsetZ); - column.downsampleFrom(source.subView(srcDataPerTrgData, ox * srcDataPerTrgData, oz * srcDataPerTrgData)); - } - } - } - else - { - LodUtil.assertNotReach(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index b276882b0..a0d391554 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -165,19 +165,23 @@ public class FullDataPointIdMap /** * Adds each entry from the given map to this map. + * + * Note: when using this function be careful about re-mapping the + * same data source multiple times. + * Doing so may cause indexOutOfBounds issues. * * @return an array of each added entry's ID in this map in order */ - public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap target) + public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap inputMap) { try { - LOGGER.trace("merging {" + this.pos + ", " + this.entryList.size() + "} and {" + target.pos + ", " + target.entryList.size() + "}"); + LOGGER.trace("merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); - target.readWriteLock.readLock().lock(); + inputMap.readWriteLock.readLock().lock(); this.readWriteLock.writeLock().lock(); - ArrayList entriesToMerge = target.entryList; + ArrayList entriesToMerge = inputMap.entryList; int[] remappedEntryIds = new int[entriesToMerge.size()]; for (int i = 0; i < entriesToMerge.size(); i++) { @@ -191,9 +195,9 @@ public class FullDataPointIdMap finally { this.readWriteLock.writeLock().unlock(); - target.readWriteLock.readLock().unlock(); + inputMap.readWriteLock.readLock().unlock(); - LOGGER.trace("finished merging {" + this.pos + ", " + this.entryList.size() + "} and {" + target.pos + ", " + target.entryList.size() + "}"); + LOGGER.trace("finished merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java deleted file mode 100644 index dcc02e536..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; - -import com.seibel.distanthorizons.core.pos.DhChunkPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; - -/** - * A more specific version of {@link FullDataArrayAccessor} - * that only contains full data for a single chunk. - * - * @see FullDataPointUtil - */ -public class ChunkSizedFullDataAccessor extends FullDataArrayAccessor -{ - public final DhChunkPos chunkPos; - public final DhSectionPos sectionPos; - - // TODO replace this var with LodUtil.BLOCK_DETAIL_LEVEL - public final byte detailLevel = LodUtil.BLOCK_DETAIL_LEVEL; - - - - public ChunkSizedFullDataAccessor(DhChunkPos chunkPos) - { - super(new FullDataPointIdMap(new DhSectionPos(chunkPos)), - new long[LodUtil.CHUNK_WIDTH * LodUtil.CHUNK_WIDTH][0], - LodUtil.CHUNK_WIDTH); - - this.chunkPos = chunkPos; - // TODO the fact this is using a LodUtil detail level instead of the DhSectionPos detail level may cause confusion and trouble down the line - this.sectionPos = new DhSectionPos(LodUtil.CHUNK_DETAIL_LEVEL, this.chunkPos.x, this.chunkPos.z); - } - - - - public void setSingleColumn(long[] data, int xRelative, int zRelative) { this.dataArrays[xRelative * LodUtil.CHUNK_WIDTH + zRelative] = data; } - - public long nonEmptyCount() - { - long count = 0; - for (long[] data : this.dataArrays) - { - if (data.length != 0) - { - count += 1; - } - } - return count; - } - - public long emptyCount() { return (LodUtil.CHUNK_WIDTH * LodUtil.CHUNK_WIDTH) - this.nonEmptyCount(); } - - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public String toString() { return this.chunkPos + " " + this.nonEmptyCount(); } - -} \ No newline at end of file diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java index db8eec620..123a29653 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java @@ -23,15 +23,14 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; /** * Contains Full Data points and basic methods for getting and setting them.
* Can be used standalone or as the base for Full data sources. * * @see CompleteFullDataSource - * @see LowDetailIncompleteFullDataSource */ +@Deprecated public class FullDataArrayAccessor implements IFullDataAccessor { protected final FullDataPointIdMap mapping; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java index 6f45e1707..62cd58cc4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java @@ -19,8 +19,8 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.FullDataPointUtil; @@ -28,12 +28,12 @@ import java.util.Iterator; /** * Contains raw full data points, which must be interpreted by the {@link FullDataPointUtil}.
- * Often used by {@link IFullDataSource}'s. + * Often used by {@link CompleteFullDataSource}'s. * - * @see IFullDataSource * @see FullDataArrayAccessor * @see FullDataPointUtil */ +@Deprecated public interface IFullDataAccessor { FullDataPointIdMap getMapping(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java deleted file mode 100644 index 406d64b97..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.google.common.collect.HashMultimap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.locks.ReentrantLock; - -public abstract class AbstractFullDataSourceLoader -{ - public static final HashMultimap, AbstractFullDataSourceLoader> LOADER_REGISTRY = HashMultimap.create(); - public static final HashMap> DATATYPE_REGISTRY = new HashMap<>(); - - - public final Class fullDataSourceClass; - - public final String datatype; - public final byte[] loaderSupportedVersions; - - /** used when pooling data sources */ - private final ArrayList cachedSources = new ArrayList<>(); - private final ReentrantLock cacheLock = new ReentrantLock(); - - - - //=============// - // constructor // - //=============// - - public AbstractFullDataSourceLoader(Class fullDataSourceClass, String datatype, byte[] loaderSupportedVersions) - { - this.datatype = datatype; - this.loaderSupportedVersions = loaderSupportedVersions; - Arrays.sort(loaderSupportedVersions); // sort to allow fast access - this.fullDataSourceClass = fullDataSourceClass; - - if (DATATYPE_REGISTRY.containsKey(datatype) && DATATYPE_REGISTRY.get(datatype) != fullDataSourceClass) - { - throw new IllegalArgumentException("Loader for datatype: [" + datatype + "] already registered with different class: " - + DATATYPE_REGISTRY.get(datatype) + " != " + fullDataSourceClass); - } - - Set loaders = LOADER_REGISTRY.get(fullDataSourceClass); - if (loaders.stream().anyMatch(other -> - { - // see if any loaderSupportsVersion conflicts with this one - for (byte otherVer : other.loaderSupportedVersions) - { - if (Arrays.binarySearch(loaderSupportedVersions, otherVer) >= 0) - { - return true; - } - } - return false; - })) - { - throw new IllegalArgumentException("Loader for class " + fullDataSourceClass + " that supports one of the version in " - + Arrays.toString(loaderSupportedVersions) + " already registered!"); - } - - DATATYPE_REGISTRY.put(datatype, fullDataSourceClass); - LOADER_REGISTRY.put(fullDataSourceClass, this); - } - - - - //================// - // loader getters // - //================// - - public static AbstractFullDataSourceLoader getLoader(String dataType, byte dataVersion) - { - return LOADER_REGISTRY.get(DATATYPE_REGISTRY.get(dataType)).stream() - .filter(loader -> Arrays.binarySearch(loader.loaderSupportedVersions, dataVersion) >= 0) - .findFirst().orElse(null); - } - - public static AbstractFullDataSourceLoader getLoader(Class clazz, byte dataVersion) - { - return LOADER_REGISTRY.get(clazz).stream() - .filter(loader -> Arrays.binarySearch(loader.loaderSupportedVersions, dataVersion) >= 0) - .findFirst().orElse(null); - } - - - - //==================// - // abstract methods // - //==================// - - protected abstract IFullDataSource createEmptyDataSource(DhSectionPos pos); - - - - //==============// - // data loading // - //==============// - - /** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */ - public IFullDataSource loadTemporaryDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException - { - IFullDataSource dataSource = this.tryGetPooledSource(); - if (dataSource != null) - { - dataSource.repopulateFromStream(dto, dto.getInputStream(), level); - } - else - { - dataSource = this.loadDataSource(dto, level); - } - - return dataSource; - } - - /** - * Can return null if any of the requirements aren't met. - * - * @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down - */ - public IFullDataSource loadDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException - { - IFullDataSource dataSource = this.createEmptyDataSource(dto.pos); - dataSource.populateFromStream(dto, dto.getInputStream(), level); - return dataSource; - } - - - - //=====================// - // data source pooling // - //=====================// - - /** @return null if no pooled source exists */ - public IFullDataSource tryGetPooledSource() - { - try - { - this.cacheLock.lock(); - - int index = this.cachedSources.size() - 1; - if (index == -1) - { - return null; - } - else - { - return this.cachedSources.remove(index); - } - } - finally - { - this.cacheLock.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link AbstractFullDataSourceLoader#tryGetPooledSource} is called. - */ - public void returnPooledDataSource(IFullDataSource dataSource) - { - if (dataSource == null) - { - return; - } - else if (dataSource.getClass() != this.fullDataSourceClass) - { - return; - } - else if (this.cachedSources.size() > 25) - { - return; - } - - try - { - this.cacheLock.lock(); - this.cachedSources.add(dataSource); - } - finally - { - this.cacheLock.unlock(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java deleted file mode 100644 index d9088d20d..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; - -public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader -{ - public CompleteFullDataSourceLoader() { super(CompleteFullDataSource.class, CompleteFullDataSource.DATA_TYPE_NAME, new byte[]{CompleteFullDataSource.DATA_FORMAT_VERSION}); } - - @Override - protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return CompleteFullDataSource.createEmpty(pos); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java deleted file mode 100644 index 9dcfb4f58..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource; - -public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader -{ - public HighDetailIncompleteFullDataSourceLoader() { super(HighDetailIncompleteFullDataSource.class, HighDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{HighDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } - - @Override - protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return HighDetailIncompleteFullDataSource.createEmpty(pos); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java deleted file mode 100644 index 49793c958..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; - -public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader -{ - public LowDetailIncompleteFullDataSourceLoader() { super(LowDetailIncompleteFullDataSource.class, LowDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{LowDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } - - @Override - protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return LowDetailIncompleteFullDataSource.createEmpty(pos); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 1a1e4f885..4ed5e4cc7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -20,17 +20,13 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; +import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhBlockPos2D; -import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -47,10 +43,8 @@ import java.util.Arrays; * This data source contains every datapoint over its given {@link DhSectionPos}. * * @see FullDataPointUtil - * @see LowDetailIncompleteFullDataSource - * @see HighDetailIncompleteFullDataSource */ -public class CompleteFullDataSource extends FullDataArrayAccessor implements IFullDataSource, IStreamableFullDataSource +public class CompleteFullDataSource extends FullDataArrayAccessor implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -60,8 +54,15 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu public static final byte DATA_FORMAT_VERSION = 3; public static final String DATA_TYPE_NAME = "CompleteFullDataSource"; - @Override - public String getDataTypeName() { return DATA_TYPE_NAME; } + + /** + * This is the byte put between different sections in the binary save file. + * The presence and absence of this byte indicates if the file is correctly formatted. + */ + private static final int DATA_GUARD_BYTE = 0xFFFFFFFF; + /** indicates the binary save file represents an empty data source */ + private static final int NO_DATA_FLAG_BYTE = 0x00000001; + private DhSectionPos sectionPos; @@ -92,11 +93,100 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu - //=================// - // stream handling // - //=================// + //======// + // data // + //======// + + @Deprecated + @Override + public void update(NewFullDataSource dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); } + + + + //=====================// + // setters and getters // + //=====================// @Override + public DhSectionPos getKey() { return this.sectionPos; } + + @Override + public DhSectionPos getSectionPos() { return this.sectionPos; } + + public void resizeDataStructuresForRepopulation(DhSectionPos pos) + { + // no data structures need to be changed, only the source's position + this.sectionPos = pos; + } + + @Override + public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } + + @Override + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } + + @Override + public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } + @Override + public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { return this.worldGenStep; } + + public boolean isEmpty() { return this.isEmpty; } + + + + //=================// + // stream handling // + //=================// + + /** + * Clears and then overwrites any data in this object with the data from the given file and stream. + * This is expected to be used with an existing {@link CompleteFullDataSource} and can be used in place of a constructor to reuse an existing {@link CompleteFullDataSource} object. + */ + public void repopulateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + { + // clear/overwrite the old data + this.resizeDataStructuresForRepopulation(dto.pos); + this.getMapping().clear(dto.pos); + + // set the new data + this.populateFromStream(dto, inputStream, level); + } + + /** + * Overwrites any data in this object with the data from the given file and stream. + * This is expected to be used with an empty {@link CompleteFullDataSource} and functions similar to a constructor. + */ + public void populateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + { + FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level); + this.setSourceSummaryData(summaryData); + + + long[][] dataPoints = this.readDataPoints(summaryData.dataWidth, inputStream); + if (dataPoints == null) + { + return; + } + this.setDataPoints(dataPoints); + + + FullDataPointIdMap mapping = this.readIdMappings(inputStream, level.getLevelWrapper()); + this.setIdMapping(mapping); + + } + + + // low level stream methods // + + @Deprecated + @Override + public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException + { + throw new UnsupportedOperationException("Deprecated"); + } + + /** unused, just here for reference as to how the data was written */ + @Deprecated public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException { outputStream.writeInt(this.getDataDetailLevel()); @@ -105,8 +195,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu outputStream.writeByte(this.worldGenStep.value); } - @Override - public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetail = inputStream.readInt(); if (dataDetail != dto.dataDetailLevel) @@ -137,21 +226,18 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return new FullDataSourceSummaryData(width, worldGenStep); } - public void setSourceSummaryData(FullDataSourceSummaryData summaryData) - { - this.worldGenStep = summaryData.worldGenStep; - } + public void setSourceSummaryData(FullDataSourceSummaryData summaryData) { this.worldGenStep = summaryData.worldGenStep; } - - @Override + /** unused, just here for reference as to how the data was written */ + @Deprecated public boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException { if (this.isEmpty()) { - outputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); + outputStream.writeInt(NO_DATA_FLAG_BYTE); return false; } - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(DATA_GUARD_BYTE); @@ -167,7 +253,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu // Data array content (only on non-empty columns) - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(DATA_GUARD_BYTE); for (int x = 0; x < this.width; x++) { for (int z = 0; z < this.width; z++) @@ -187,19 +273,18 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return true; } - @Override - public long[][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream dataInputStream) throws IOException + public long[][] readDataPoints(int width, DhDataInputStream dataInputStream) throws IOException { // Data array length int dataPresentFlag = dataInputStream.readInt(); - if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) + if (dataPresentFlag == NO_DATA_FLAG_BYTE) { // Section is empty return null; } - else if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) + else if (dataPresentFlag != DATA_GUARD_BYTE) { - throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + IFullDataSource.NO_DATA_FLAG_BYTE + "] or (data present) [" + IFullDataSource.DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); + throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + NO_DATA_FLAG_BYTE + "] or (data present) [" + DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); } @@ -211,7 +296,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu } else { - dataPointArrays = new long[width * width][]; + dataPointArrays = new long[width * width][]; } for (int x = 0; x < width; x++) @@ -238,7 +323,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu // check if the array start flag is present int arrayStartFlag = dataInputStream.readInt(); - if (arrayStartFlag != IFullDataSource.DATA_GUARD_BYTE) + if (arrayStartFlag != DATA_GUARD_BYTE) { throw new IOException("invalid data length end guard"); } @@ -258,7 +343,6 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return dataPointArrays; } - @Override public void setDataPoints(long[][] dataPoints) { LodUtil.assertTrue(this.dataArrays.length == dataPoints.length, "Data point array length mismatch."); @@ -268,183 +352,47 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu } - @Override + /** unused, just here for reference as to how the data was written */ + @Deprecated public void writeIdMappings(DhDataOutputStream outputStream) throws IOException { - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(DATA_GUARD_BYTE); this.mapping.serialize(outputStream); } - @Override - public FullDataPointIdMap readIdMappings(long[][] dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException { int guardByte = inputStream.readInt(); - if (guardByte != IFullDataSource.DATA_GUARD_BYTE) + if (guardByte != DATA_GUARD_BYTE) { throw new IOException("Invalid data content end guard for ID mapping"); } return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper); } - @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } - //======// - // data // - //======// + //================// + // helper classes // + //================// - @Override - public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.get(relativeX, relativeZ); } - @Override - public SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ) { return this.get(relativeX, relativeZ); } - - @Override - public void update(ChunkSizedFullDataAccessor chunkDataView) + /** + * This holds information that is relevant to the entire source and isn't stored in the data points.
+ * Example: minimum height, detail level, source type, etc. + */ + private static class FullDataSourceSummaryData { - LodUtil.assertTrue(this.sectionPos.overlapsExactly(chunkDataView.getSectionPos())); - if (this.getDataDetailLevel() == LodUtil.BLOCK_DETAIL_LEVEL) + public final int dataWidth; + public EDhApiWorldGenerationStep worldGenStep; + + + public FullDataSourceSummaryData(int dataWidth, EDhApiWorldGenerationStep worldGenStep) { - DhBlockPos2D chunkBlockPos = new DhBlockPos2D(chunkDataView.chunkPos.x * LodUtil.CHUNK_WIDTH, chunkDataView.chunkPos.z * LodUtil.CHUNK_WIDTH); - DhBlockPos2D blockOffset = chunkBlockPos.subtract(this.sectionPos.getMinCornerLodPos().getCornerBlockPos()); - LodUtil.assertTrue(blockOffset.x >= 0 && blockOffset.x < WIDTH && blockOffset.z >= 0 && blockOffset.z < WIDTH); - this.isEmpty = false; - - chunkDataView.shadowCopyTo(this.subView(LodUtil.CHUNK_WIDTH, blockOffset.x, blockOffset.z)); - - // DEBUG ASSERTION - { - for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++) - { - for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++) - { - SingleColumnFullDataAccessor column = this.get(x + blockOffset.x, z + blockOffset.z); - LodUtil.assertTrue(column.doesColumnExist()); - } - } - } - } - else if (this.getDataDetailLevel() < LodUtil.CHUNK_DETAIL_LEVEL) - { - int dataPerFull = 1 << this.getDataDetailLevel(); - int fullSize = LodUtil.CHUNK_WIDTH / dataPerFull; - DhLodPos dataOffset = chunkDataView.getSectionPos().getMinCornerLodPos(this.getDataDetailLevel()); - DhLodPos baseOffset = this.sectionPos.getMinCornerLodPos(this.getDataDetailLevel()); - - int offsetX = dataOffset.x - baseOffset.x; - int offsetZ = dataOffset.z - baseOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < WIDTH && offsetZ >= 0 && offsetZ < WIDTH); - - this.isEmpty = false; - for (int xOffset = 0; xOffset < fullSize; xOffset++) - { - for (int zOffset = 0; zOffset < fullSize; zOffset++) - { - SingleColumnFullDataAccessor column = this.get(xOffset + offsetX, zOffset + offsetZ); - column.downsampleFrom(chunkDataView.subView(dataPerFull, xOffset * dataPerFull, zOffset * dataPerFull)); - } - } - } - else if (this.getDataDetailLevel() >= LodUtil.CHUNK_DETAIL_LEVEL) - { - //FIXME: TEMPORARY - int chunkPerFull = 1 << (this.getDataDetailLevel() - LodUtil.CHUNK_DETAIL_LEVEL); - if (chunkDataView.chunkPos.x % chunkPerFull != 0 || chunkDataView.chunkPos.z % chunkPerFull != 0) - { - return; - } - - DhLodPos baseOffset = this.sectionPos.getMinCornerLodPos(this.getDataDetailLevel()); - DhSectionPos dataOffset = chunkDataView.getSectionPos().convertNewToDetailLevel(this.getDataDetailLevel()); - - int offsetX = dataOffset.getX() - baseOffset.x; - int offsetZ = dataOffset.getZ() - baseOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < WIDTH && offsetZ >= 0 && offsetZ < WIDTH); - - this.isEmpty = false; - chunkDataView.get(0, 0).deepCopyTo(this.get(offsetX, offsetZ)); - } - else - { - LodUtil.assertNotReach(); - //TODO + this.dataWidth = dataWidth; + this.worldGenStep = worldGenStep; } } - - - //================// - // helper methods // - //================// - - /** Returns whether data at the given posToWrite can effect the target region file at posToTest. */ - public static boolean firstDataPosCanAffectSecond(DhSectionPos posToWrite, DhSectionPos posToTest) - { - if (!posToWrite.overlapsExactly(posToTest)) - { - // the testPosition is outside the writePosition - return false; - } - else if (posToTest.getDetailLevel() > posToWrite.getDetailLevel()) - { - // the testPosition is larger (aka is less detailed) than the writePosition, - // more detailed sections shouldn't be updated by lower detail sections - return false; - } - else if (posToWrite.getDetailLevel() - posToTest.getDetailLevel() <= SECTION_SIZE_OFFSET) - { - // if the difference in detail levels is very large, the posToWrite - // may be skipped, due to how we sample large detail levels by only - // getting the corners. - - // In this case the difference isn't very large, so return true - return true; - } - else - { - // the difference in detail levels is very large, - // check if the posToWrite is in a corner of posToTest - byte sectPerData = (byte) BitShiftUtil.powerOfTwo(posToWrite.getDetailLevel() - posToTest.getDetailLevel() - SECTION_SIZE_OFFSET); - LodUtil.assertTrue(sectPerData != 0); - return posToTest.getX() % sectPerData == 0 && posToTest.getZ() % sectPerData == 0; - } - } - - - - //=====================// - // setters and getters // - //=====================// - - @Override - public DhSectionPos getKey() { return this.sectionPos; } - - @Override - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public void resizeDataStructuresForRepopulation(DhSectionPos pos) - { - // no data structures need to be changed, only the source's position - this.sectionPos = pos; - } - - @Override - public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - - @Override - public boolean isEmpty() { return this.isEmpty; } - @Override - public void markNotEmpty() { this.isEmpty = false; } - - @Override - public int getWidthInDataPoints() { return this.width; } - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java deleted file mode 100644 index d1a2959c6..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ /dev/null @@ -1,586 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources; - -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; -import org.apache.logging.log4j.Logger; - -import java.io.*; -import java.util.BitSet; - -/** - * Used for small incomplete LOD blocks.
- * Handles incomplete full data with a detail level equal to or lower than - * {@link HighDetailIncompleteFullDataSource#MAX_SECTION_DETAIL}.

- * - * Compared to other {@link IIncompleteFullDataSource}'s, this object doesn't extend {@link FullDataArrayAccessor}, - * instead it contains several "sections" of data, represented by {@link FullDataArrayAccessor}s.

- * - * Formerly "SparseFullDataSource". - * - * @see LowDetailIncompleteFullDataSource - * @see CompleteFullDataSource - * @see FullDataPointUtil - */ -public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSource, IStreamableFullDataSource -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - // TODO James would like to rename, comment, and potentially remove some of these constants. - // But he doesn't currently have the understanding to do so. - public static final byte SPARSE_UNIT_DETAIL = LodUtil.CHUNK_DETAIL_LEVEL; - public static final byte SPARSE_UNIT_SIZE = (byte) BitShiftUtil.powerOfTwo(SPARSE_UNIT_DETAIL); - - public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - public static final int SECTION_SIZE = (byte) BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); - /** aka max detail level */ - public static final byte MAX_SECTION_DETAIL = SECTION_SIZE_OFFSET + SPARSE_UNIT_DETAIL; - - public static final byte DATA_FORMAT_VERSION = 3; - public static final String DATA_TYPE_NAME = "HighDetailIncompleteFullDataSource"; - @Override - public String getDataTypeName() { return DATA_TYPE_NAME; } - - - protected final FullDataPointIdMap mapping; - private DhSectionPos sectionPos; - private FullDataArrayAccessor[] sparseData; - private DhLodPos chunkPos; - - public int sectionCount; - public int dataPointsPerSection; - public boolean isEmpty = true; - public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY; - private boolean isPromoted = false; - - - - //==============// - // constructors // - //==============// - - public static HighDetailIncompleteFullDataSource createEmpty(DhSectionPos pos) { return new HighDetailIncompleteFullDataSource(pos); } - private HighDetailIncompleteFullDataSource(DhSectionPos sectionPos) - { - LodUtil.assertTrue(sectionPos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(sectionPos.getDetailLevel() <= MAX_SECTION_DETAIL); - - this.sectionPos = sectionPos; - this.sectionCount = BitShiftUtil.powerOfTwo(sectionPos.getDetailLevel() - SPARSE_UNIT_DETAIL); - this.dataPointsPerSection = SECTION_SIZE / this.sectionCount; - - this.sparseData = new FullDataArrayAccessor[this.sectionCount * this.sectionCount]; - this.chunkPos = sectionPos.getMinCornerLodPos(SPARSE_UNIT_DETAIL); - this.mapping = new FullDataPointIdMap(sectionPos); - } - - protected HighDetailIncompleteFullDataSource(DhSectionPos sectionPos, FullDataPointIdMap mapping, FullDataArrayAccessor[] data) - { - LodUtil.assertTrue(sectionPos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(sectionPos.getDetailLevel() <= MAX_SECTION_DETAIL); - - this.sectionPos = sectionPos; - this.sectionCount = 1 << (byte) (sectionPos.getDetailLevel() - SPARSE_UNIT_DETAIL); - this.dataPointsPerSection = SECTION_SIZE / this.sectionCount; - - LodUtil.assertTrue(this.sectionCount * this.sectionCount == data.length); - this.sparseData = data; - this.chunkPos = sectionPos.getMinCornerLodPos(SPARSE_UNIT_DETAIL); - this.isEmpty = false; - this.mapping = mapping; - } - - - - //=================// - // stream handling // - //=================// - - - @Override - public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream dataOutputStream) throws IOException - { - dataOutputStream.writeShort(this.getDataDetailLevel()); - dataOutputStream.writeShort(SPARSE_UNIT_DETAIL); - dataOutputStream.writeInt(SECTION_SIZE); - dataOutputStream.writeInt(level.getMinY()); - dataOutputStream.writeByte(this.worldGenStep.value); - - } - @Override - public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException - { - LodUtil.assertTrue(dto.pos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(dto.pos.getDetailLevel() <= MAX_SECTION_DETAIL); - - int dataDetailLevel = inputStream.readShort(); - if (dataDetailLevel != dto.dataDetailLevel) - { - throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.dataDetailLevel+"]"); - } - - // confirm that the detail level is correct - int sparseDetail = inputStream.readShort(); - if (sparseDetail != SPARSE_UNIT_DETAIL) - { - throw new IOException("Unexpected sparse detail level: ["+sparseDetail+"] != ["+SPARSE_UNIT_DETAIL+"]"); - } - - // confirm the scale of the data points is correct - int sectionSize = inputStream.readInt(); - if (sectionSize != SECTION_SIZE) - { - throw new IOException("Section size mismatch: ["+sectionSize+"] != ["+SECTION_SIZE+"] (Currently only 1 section size is supported)"); - } - - int minY = inputStream.readInt(); - if (minY != level.getMinY()) - { - LOGGER.warn("Data minY mismatch: [" + minY + "] != [" + level.getMinY() + "]. Will ignore data's y level"); - } - - EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte()); - if (worldGenStep == null) - { - worldGenStep = EDhApiWorldGenerationStep.SURFACE; - LOGGER.warn("Missing WorldGenStep, defaulting to: " + worldGenStep.name()); - } - - - return new FullDataSourceSummaryData(-1, worldGenStep); - } - public void setSourceSummaryData(FullDataSourceSummaryData summaryData) { this.worldGenStep = summaryData.worldGenStep; } - - - @Override - public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException - { - if (this.isEmpty) - { - dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); - return false; - } - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - - - // sparse array existence bitset - BitSet dataArrayIndexHasData = new BitSet(this.sparseData.length); - for (int i = 0; i < this.sparseData.length; i++) - { - dataArrayIndexHasData.set(i, this.sparseData[i] != null); - } - byte[] bytes = dataArrayIndexHasData.toByteArray(); - dataOutputStream.writeInt(bytes.length); - dataOutputStream.write(bytes); - - - // Data array content (only non-empty data is written) - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - for (int dataArrayIndex = dataArrayIndexHasData.nextSetBit(0); - dataArrayIndex >= 0; - dataArrayIndex = dataArrayIndexHasData.nextSetBit(dataArrayIndex + 1)) - { - // column data length - FullDataArrayAccessor array = this.sparseData[dataArrayIndex]; - LodUtil.assertTrue(array != null); - for (int x = 0; x < array.width(); x++) - { - for (int z = 0; z < array.width(); z++) - { - SingleColumnFullDataAccessor columnAccessor = array.get(x, z); - int columnLength = 0; - if (columnAccessor != null) - { - columnLength = columnAccessor.getSingleLength(); - } - dataOutputStream.writeInt(columnLength); - } - } - - // column data - for (int x = 0; x < array.width(); x++) - { - for (int z = 0; z < array.width(); z++) - { - SingleColumnFullDataAccessor column = array.get(x, z); - LodUtil.assertTrue(column.getMapping() == this.mapping); // the mappings must be exactly equal! - - if (column.doesColumnExist()) - { - long[] rawDataPoints = column.getRaw(); - for (long dataPoint : rawDataPoints) - { - dataOutputStream.writeLong(dataPoint); - } - } - } - } - } - - - return true; - } - @Override - public long[][][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException - { - // calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize - // TODO these values should be constant, should we still be calculating them like this? - int chunks = BitShiftUtil.powerOfTwo(dto.pos.getDetailLevel() - SPARSE_UNIT_DETAIL); - int dataPointsPerChunk = SECTION_SIZE / chunks; - - - // check if this file has any data - int dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) - { - // this file is empty - return null; - } - else if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + IFullDataSource.NO_DATA_FLAG_BYTE + "] or (data present) [" + IFullDataSource.DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); - } - - - // get the number of columns (IE the bitSet from before) - int numberOfDataColumns = inputStream.readInt(); - // validate the number of data columns - int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent? - if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns) - { - throw new IOException(LodUtil.formatLog("Sparse Flag BitSet size outside reasonable range: {} (expects {} to {})", - numberOfDataColumns, 1, maxNumberOfDataColumns)); - } - - // read in the presence of each data column - byte[] bytes = new byte[numberOfDataColumns]; - inputStream.readFully(bytes, 0, numberOfDataColumns); - BitSet dataArrayIndexHasData = BitSet.valueOf(bytes); - - - - //====================// - // Data array content // - //====================// - - // (only on non-empty columns) - int dataArrayStartByte = inputStream.readInt(); - // confirm the column data is starting - if (dataArrayStartByte != IFullDataSource.DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("invalid data length end guard"); - } - - - // read in each column that has data written to it - long[][][] rawFullDataArrays = new long[chunks * chunks][][]; - for (int fullDataIndex = dataArrayIndexHasData.nextSetBit(0); - fullDataIndex >= 0 && // TODO why does this happen? - fullDataIndex < rawFullDataArrays.length; - fullDataIndex = dataArrayIndexHasData.nextSetBit(fullDataIndex + 1)) - { - long[][] dataColumn = new long[dataPointsPerChunk * dataPointsPerChunk][]; - - // get the column data lengths - rawFullDataArrays[fullDataIndex] = dataColumn; - for (int x = 0; x < dataColumn.length; x++) - { - // this should be zero if the column doesn't have any data - int dataColumnLength = inputStream.readInt(); - dataColumn[x] = new long[dataColumnLength]; - } - - // get the column data - for (int x = 0; x < dataColumn.length; x++) - { - if (dataColumn[x].length != 0) - { - // read in the data columns - for (int z = 0; z < dataColumn[x].length; z++) - { - dataColumn[x][z] = inputStream.readLong(); - } - } - } - } - - - return rawFullDataArrays; - } - @Override - public void setDataPoints(long[][][] dataPoints) - { - LodUtil.assertTrue(this.sparseData.length == dataPoints.length, "Data point array length mismatch."); - - this.isEmpty = false; - - - for (int arrayAccessorIndex = 0; arrayAccessorIndex < dataPoints.length; arrayAccessorIndex++) - { - if (dataPoints[arrayAccessorIndex] == null) - { - this.sparseData[arrayAccessorIndex] = null; - } - else if (this.sparseData[arrayAccessorIndex] == null) - { - int width = (int) Math.sqrt(dataPoints[arrayAccessorIndex].length); - this.sparseData[arrayAccessorIndex] = new FullDataArrayAccessor(this.mapping, dataPoints[arrayAccessorIndex], width); - } - else - { - for (int dataPointColIndex = 0; dataPointColIndex < dataPoints[arrayAccessorIndex].length; dataPointColIndex++) - { - long[] incomingColumn = dataPoints[arrayAccessorIndex][dataPointColIndex]; - long[] destinationColumn = this.sparseData[arrayAccessorIndex].get(dataPointColIndex).getRaw(); - - // use the existing arrays if possible - if (incomingColumn.length == destinationColumn.length) - { - System.arraycopy(incomingColumn, 0, destinationColumn, 0, incomingColumn.length); - } - else - { - this.sparseData[arrayAccessorIndex].get(dataPointColIndex).setNew(incomingColumn); - } - } - } - } - } - - - @Override - public FullDataPointIdMap readIdMappings(long[][][] dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException - { - // mark the start of the ID data - int idMappingStartByte = inputStream.readInt(); - if (idMappingStartByte != DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("invalid data content end guard"); - } - - // deserialize the ID data - return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper); - } - @Override - public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException - { - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(dataOutputStream); - } - @Override - public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } - - - - //======// - // data // - //======// - - @Override - public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, false); } - @Override - public SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, true); } - private SingleColumnFullDataAccessor tryGetOrCreate(int relativeX, int relativeZ, boolean createIfMissing) - { - LodUtil.assertTrue(relativeX >= 0 && relativeX < SECTION_SIZE && relativeZ >= 0 && relativeZ < SECTION_SIZE); - int chunkX = relativeX / this.dataPointsPerSection; - int chunkZ = relativeZ / this.dataPointsPerSection; - FullDataArrayAccessor accessor = this.sparseData[chunkX * this.sectionCount + chunkZ]; - if (accessor == null) - { - if (createIfMissing) - { - // create the missing data so the following get() will succeed - accessor = new FullDataArrayAccessor(this.mapping, new long[this.dataPointsPerSection * this.dataPointsPerSection][], this.dataPointsPerSection); - this.sparseData[chunkX * this.sectionCount + chunkZ] = accessor; - } - else - { - return null; - } - } - - return accessor.get(relativeX % this.dataPointsPerSection, relativeZ % this.dataPointsPerSection); - } - - - - //=========// - // getters // - //=========// - - @Override - public DhSectionPos getKey() { return this.sectionPos; } - - @Override - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public void resizeDataStructuresForRepopulation(DhSectionPos pos) - { - // update the position - this.sectionPos = pos; - this.sectionCount = BitShiftUtil.powerOfTwo(this.sectionPos.getDetailLevel() - SPARSE_UNIT_DETAIL); - this.dataPointsPerSection = SECTION_SIZE / this.sectionCount; - - this.chunkPos = this.sectionPos.getMinCornerLodPos(SPARSE_UNIT_DETAIL); - - - // update the data container - int dataPointCount = this.sectionCount * this.sectionCount; - if (this.sparseData.length != dataPointCount) - { - this.sparseData = new FullDataArrayAccessor[this.sectionCount * this.sectionCount]; - } - - } - - @Override - public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - - @Override - public FullDataPointIdMap getMapping() { return this.mapping; } - - @Override - public boolean isEmpty() { return this.isEmpty; } - @Override - public void markNotEmpty() { this.isEmpty = false; } - - @Override - public int getWidthInDataPoints() { return SECTION_SIZE; } - - - private int calculateOffset(int chunkX, int chunkZ) - { - int offsetX = chunkX - this.chunkPos.x; - int offsetZ = chunkZ - this.chunkPos.z; - LodUtil.assertTrue(offsetX >= 0 && offsetZ >= 0 && offsetX < this.sectionCount && offsetZ < this.sectionCount); - return offsetX * this.sectionCount + offsetZ; - } - - - - //=============// - // data update // - //=============// - - @Override - public void update(ChunkSizedFullDataAccessor chunkDataView) - { - int arrayOffset = this.calculateOffset(chunkDataView.chunkPos.x, chunkDataView.chunkPos.z); - FullDataArrayAccessor newArray = new FullDataArrayAccessor(this.mapping, new long[this.dataPointsPerSection * this.dataPointsPerSection][], this.dataPointsPerSection); - if (this.getDataDetailLevel() == chunkDataView.detailLevel) - { - chunkDataView.shadowCopyTo(newArray); - } - else - { - int count = this.dataPointsPerSection; - int dataPerCount = SPARSE_UNIT_SIZE / this.dataPointsPerSection; - - for (int xOffset = 0; xOffset < count; xOffset++) - { - for (int zOffset = 0; zOffset < count; zOffset++) - { - SingleColumnFullDataAccessor column = newArray.get(xOffset, zOffset); - column.downsampleFrom(chunkDataView.subView(dataPerCount, xOffset * dataPerCount, zOffset * dataPerCount)); - } - } - } - - this.isEmpty = false; - this.sparseData[arrayOffset] = newArray; - } - - - // data sampling // - - private void applyToFullDataSource(CompleteFullDataSource dataSource) - { - LodUtil.assertTrue(dataSource.getSectionPos().equals(this.sectionPos)); - LodUtil.assertTrue(dataSource.getDataDetailLevel() == this.getDataDetailLevel()); - for (int x = 0; x < this.sectionCount; x++) - { - for (int z = 0; z < this.sectionCount; z++) - { - FullDataArrayAccessor array = this.sparseData[x * this.sectionCount + z]; - if (array == null) - continue; - - // Otherwise, apply data to dataSource - dataSource.markNotEmpty(); - FullDataArrayAccessor view = dataSource.subView(this.dataPointsPerSection, x * this.dataPointsPerSection, z * this.dataPointsPerSection); - array.shadowCopyTo(view); - } - } - } - - public IFullDataSource tryPromotingToCompleteDataSource() - { - if (this.isEmpty) - { - return this; - } - - // promotion can only succeed if every data column is present - for (FullDataArrayAccessor array : this.sparseData) - { - if (array == null) - { - return this; - } - } - this.isPromoted = true; - CompleteFullDataSource fullDataSource = CompleteFullDataSource.createEmpty(this.sectionPos); - this.applyToFullDataSource(fullDataSource); - return fullDataSource; - } - - @Override - public boolean hasBeenPromoted() { return this.isPromoted; } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java deleted file mode 100644 index e80d4d813..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ /dev/null @@ -1,439 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources; - -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; -import org.apache.logging.log4j.Logger; - -import java.io.*; -import java.util.BitSet; - -/** - * Used for large incomplete LOD blocks.
- * Handles incomplete full data with a detail level higher than - * {@link HighDetailIncompleteFullDataSource#MAX_SECTION_DETAIL}.

- * - * Formerly "SpottyFullDataSource". - * - * @see HighDetailIncompleteFullDataSource - * @see CompleteFullDataSource - * @see FullDataPointUtil - */ -public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor implements IIncompleteFullDataSource, IStreamableFullDataSource -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - /** measured in dataPoints */ - public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); - - public static final byte DATA_FORMAT_VERSION = 3; - public static final String DATA_TYPE_NAME = "LowDetailIncompleteFullDataSource"; - @Override - public String getDataTypeName() { return DATA_TYPE_NAME; } - - private DhSectionPos sectionPos; - - private final BitSet isColumnNotEmpty; - - private boolean isEmpty = true; - public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY; - private boolean isPromoted = false; - - - - //==============// - // constructors // - //==============// - - public static LowDetailIncompleteFullDataSource createEmpty(DhSectionPos pos) { return new LowDetailIncompleteFullDataSource(pos); } - private LowDetailIncompleteFullDataSource(DhSectionPos sectionPos) - { - super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH); - LodUtil.assertTrue(sectionPos.getDetailLevel() > HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL); - - this.sectionPos = sectionPos; - this.isColumnNotEmpty = new BitSet(WIDTH * WIDTH); - this.worldGenStep = EDhApiWorldGenerationStep.EMPTY; - } - - private LowDetailIncompleteFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, EDhApiWorldGenerationStep worldGenStep, BitSet isColumnNotEmpty, long[][] data) - { - super(mapping, data, WIDTH); - LodUtil.assertTrue(data.length == WIDTH * WIDTH); - - this.sectionPos = pos; - this.isColumnNotEmpty = isColumnNotEmpty; - this.worldGenStep = worldGenStep; - this.isEmpty = false; - } - - - - //=================// - // stream handling // - //=================// - - - @Override - public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException - { - outputStream.writeInt(this.getDataDetailLevel()); - outputStream.writeInt(this.width); - outputStream.writeInt(level.getMinY()); - outputStream.writeByte(this.worldGenStep.value); - - } - @Override - public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException - { - int dataDetailLevel = inputStream.readInt(); - if (dataDetailLevel != dto.dataDetailLevel) - { - throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.dataDetailLevel)); - } - - int width = inputStream.readInt(); - if (width != WIDTH) - { - throw new IOException(LodUtil.formatLog("Section size mismatch: " + width + " != " + WIDTH + " (Currently only 1 section size is supported)")); - } - - int minY = inputStream.readInt(); - if (minY != level.getMinY()) - { - LOGGER.warn("Data minY mismatch: " + minY + " != " + level.getMinY() + ". Will ignore data's y level"); - } - - EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte()); - if (worldGenStep == null) - { - worldGenStep = EDhApiWorldGenerationStep.SURFACE; - LOGGER.warn("Missing WorldGenStep, defaulting to: " + worldGenStep.name()); - } - - - return new FullDataSourceSummaryData(this.width, worldGenStep); - } - public void setSourceSummaryData(FullDataSourceSummaryData summaryData) - { - this.worldGenStep = summaryData.worldGenStep; - } - - - @Override - public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException - { - if (this.isEmpty) - { - dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); - return false; - } - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - - - // data column presence - byte[] bytes = this.isColumnNotEmpty.toByteArray(); - dataOutputStream.writeInt(bytes.length); - dataOutputStream.write(bytes); - - - // Data content - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - for (int i = this.isColumnNotEmpty.nextSetBit(0); i >= 0; i = this.isColumnNotEmpty.nextSetBit(i + 1)) - { - dataOutputStream.writeByte(this.dataArrays[i].length); - for (long dataPoint : this.dataArrays[i]) - { - dataOutputStream.writeLong(dataPoint); - } - } - - - return true; - } - @Override - public StreamDataPointContainer readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException - { - // is source empty flag - int dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) - { - // Section is empty - return null; - } - else if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + IFullDataSource.NO_DATA_FLAG_BYTE + "] or (data present) [" + IFullDataSource.DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); - } - - - // data column presence - int length = inputStream.readInt(); - if (length < 0 || length > (WIDTH * WIDTH / 8 + 64) * 2) // TODO replace magic numbers or comment what they mean - { - throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})", - length, 1, WIDTH * WIDTH / 8 + 63)); - } - - byte[] bytes = new byte[length]; - inputStream.readFully(bytes, 0, length); - BitSet isColumnNotEmpty = BitSet.valueOf(bytes); - - - - // Data array content - long[][] dataPointArray = new long[WIDTH * WIDTH][]; - dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - throw new IOException("invalid spotty flag end guard"); - } - - for (int xz = isColumnNotEmpty.nextSetBit(0); xz >= 0; xz = isColumnNotEmpty.nextSetBit(xz + 1)) - { - long[] array = new long[inputStream.readByte()]; - for (int y = 0; y < array.length; y++) - { - array[y] = inputStream.readLong(); - } - dataPointArray[xz] = array; - } - - - return new StreamDataPointContainer(dataPointArray, isColumnNotEmpty); - } - @Override - public void setDataPoints(StreamDataPointContainer streamDataPointContainer) - { - long[][] dataPoints = streamDataPointContainer.dataPoints; - - // copy over the datapoints - LodUtil.assertTrue(this.dataArrays.length == dataPoints.length, "Data point array length mismatch."); - System.arraycopy(dataPoints, 0, this.dataArrays, 0, dataPoints.length); - - // overwrite the bitset - for (int i = 0; i < streamDataPointContainer.isColumnNotEmpty.length(); i++) - { - this.isColumnNotEmpty.set(i, streamDataPointContainer.isColumnNotEmpty.get(i)); - } - - this.isEmpty = false; - } - - - @Override - public void writeIdMappings(DhDataOutputStream outputStream) throws IOException - { - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(outputStream); - - } - @Override - public FullDataPointIdMap readIdMappings(StreamDataPointContainer streamDataPointContainer, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException - { - // Id mapping - int dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - throw new IOException("invalid ID mapping end guard"); - } - return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper); - } - @Override - public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } - - - - //======// - // data // - //======// - - @Override - public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, false); } - @Override - public SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, true); } - private SingleColumnFullDataAccessor tryGetOrCreate(int relativeX, int relativeZ, boolean createIfMissing) - { - int notEmptyIndex = relativeX * WIDTH + relativeZ; - boolean columnEmpty = this.isColumnNotEmpty.get(notEmptyIndex); - - // "create" the missing column if necessary - if (columnEmpty && createIfMissing) - { - this.isColumnNotEmpty.set(notEmptyIndex, true); - columnEmpty = false; - } - - return !columnEmpty ? this.get(relativeX, relativeZ) : null; - } - - - - //=====================// - // getters and setters // - //=====================// - - @Override - public DhSectionPos getKey() { return this.sectionPos; } - - @Override - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public void resizeDataStructuresForRepopulation(DhSectionPos pos) - { - // no data structures need to be changed, only the source's position - this.sectionPos = pos; - } - - @Override - public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - - @Override - public boolean isEmpty() { return this.isEmpty; } - @Override - public void markNotEmpty() { this.isEmpty = false; } - - @Override - public int getWidthInDataPoints() { return WIDTH; } - - - - //===============// - // Data updating // - //===============// - - @Override - public void update(ChunkSizedFullDataAccessor data) - { - LodUtil.assertTrue(this.sectionPos.overlapsExactly(data.getSectionPos())); - - if (this.getDataDetailLevel() >= 4) - { - //FIXME: TEMPORARY - int chunkPerFull = 1 << (this.getDataDetailLevel() - 4); - if (data.chunkPos.x % chunkPerFull != 0 || data.chunkPos.z % chunkPerFull != 0) - { - return; - } - - DhLodPos baseOffset = this.sectionPos.getMinCornerLodPos(this.getDataDetailLevel()); - DhSectionPos dataOffset = data.getSectionPos().convertNewToDetailLevel(this.getDataDetailLevel()); - int offsetX = dataOffset.getX() - baseOffset.x; - int offsetZ = dataOffset.getZ() - baseOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < WIDTH && offsetZ >= 0 && offsetZ < WIDTH); - this.isEmpty = false; - - SingleColumnFullDataAccessor columnFullDataAccessor = this.get(offsetX, offsetZ); - data.get(0, 0).deepCopyTo(columnFullDataAccessor); - - this.isColumnNotEmpty.set(offsetX * WIDTH + offsetZ, columnFullDataAccessor.doesColumnExist()); - } - else - { - LodUtil.assertNotReach(); - //TODO; - } - - } - - @Override - public IFullDataSource tryPromotingToCompleteDataSource() - { - // promotion can only be completed if every column has data - if (this.isEmpty) - { - return this; - } - else if (this.isColumnNotEmpty.cardinality() != WIDTH * WIDTH) - { - return this; - } - this.isPromoted = true; - return new CompleteFullDataSource(this.sectionPos, this.mapping, this.dataArrays); - } - - @Override - public boolean hasBeenPromoted() { return this.isPromoted; } - - - - //================// - // helper classes // - //================// - - /** used when reading the datapoints to and from the {@link IStreamableFullDataSource} */ - public static class StreamDataPointContainer - { - public long[][] dataPoints; - public BitSet isColumnNotEmpty; - - public StreamDataPointContainer(long[][] dataPoints, BitSet isColumnNotEmpty) - { - this.dataPoints = dataPoints; - this.isColumnNotEmpty = isColumnNotEmpty; - } - - } - - - - //========// - // unused // - //========// - - public static boolean neededForPosition(DhSectionPos posToWrite, DhSectionPos posToTest) - { - if (!posToWrite.overlapsExactly(posToTest)) - return false; - if (posToTest.getDetailLevel() > posToWrite.getDetailLevel()) - return false; - if (posToWrite.getDetailLevel() - posToTest.getDetailLevel() <= SECTION_SIZE_OFFSET) - return true; - byte sectPerData = (byte) (1 << (posToWrite.getDetailLevel() - posToTest.getDetailLevel() - SECTION_SIZE_OFFSET)); - return posToTest.getX() % sectPerData == 0 && posToTest.getZ() % sectPerData == 0; - } - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java new file mode 100644 index 000000000..8eddbcc50 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -0,0 +1,447 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.dataObjects.fullData.sources; + +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; +import com.seibel.distanthorizons.core.file.IDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; +import com.seibel.distanthorizons.coreapi.ModInfo; +import org.apache.logging.log4j.Logger; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.concurrent.locks.ReentrantLock; + +/** + * This data source contains every datapoint over its given {@link DhSectionPos}. + * + * TODO create a child object that extends AutoClosable + * that can be pooled so we don't have GC overhead + * + * @see FullDataPointUtil + * @see CompleteFullDataSource + */ +public class NewFullDataSource implements IDataSource +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + /** measured in data columns */ + public static final int WIDTH = 64; + + public static final byte DATA_FORMAT_VERSION = 1; + + + + // TODO make these fields private + private DhSectionPos pos; + @Override + public DhSectionPos getKey() { return this.pos; } + + public long lastModifiedUnixDateTime; + public long createdUnixDateTime; + + public int levelMinY; + + /** + * stores how far each column has been generated should start with {@link EDhApiWorldGenerationStep#EMPTY} + * @see EDhApiWorldGenerationStep + */ + public byte[] columnGenerationSteps; + + /** stored x/z, y */ + public long[][] dataPoints; + private boolean isEmpty; + + private FullDataPointIdMap mapping; + public FullDataPointIdMap getMapping() { return this.mapping; } + + public boolean applyToParent = false; + + + + //==============// + // constructors // + //==============// + + public static NewFullDataSource createEmpty(DhSectionPos pos) { return new NewFullDataSource(pos); } + private NewFullDataSource(DhSectionPos pos) + { + this.pos = pos; + this.dataPoints = new long[WIDTH * WIDTH][]; + this.mapping = new FullDataPointIdMap(pos); + this.isEmpty = true; + + // doesn't need to be populated since nothing has been generated yet + // the default value of all 0's is adequate + this.columnGenerationSteps = new byte[WIDTH * WIDTH]; + } + + public static NewFullDataSource createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new NewFullDataSource(pos, mapping, data, columnGenerationStep); } + private NewFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) + { + LodUtil.assertTrue(data.length == WIDTH * WIDTH); + + this.pos = pos; + this.dataPoints = data; + this.mapping = mapping; + this.isEmpty = false; + + this.columnGenerationSteps = columnGenerationSteps; + } + + public static NewFullDataSource createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } + + public static NewFullDataSource createFromCompleteDataSource(CompleteFullDataSource legacyData) + { + byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; + long[][] dataPoints = new long[WIDTH * WIDTH][]; + for (int x = 0; x < WIDTH; x++) + { + for (int z = 0; z < WIDTH; z++) + { + int index = relativePosToIndex(x, z); + + SingleColumnFullDataAccessor accessor = legacyData.get(x, z); + + if (accessor.doesColumnExist()) + { + dataPoints[index] = accessor.getRaw(); + columnGenerationSteps[index] = legacyData.getWorldGenStep().value; + } + } + } + + return NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.getMapping(), dataPoints, columnGenerationSteps); + } + + + + //======// + // data // + //======// + + public SingleColumnFullDataAccessor get(int relX, int relZ) { return new SingleColumnFullDataAccessor(this.mapping, this.dataPoints, relativePosToIndex(relX, relZ)); } + + @Override + public void update(NewFullDataSource inputDataSource, @Nullable IDhLevel level) { this.update(inputDataSource); } + public void update(NewFullDataSource inputDataSource) + { + byte thisDetailLevel = this.pos.getDetailLevel(); + byte inputDetailLevel = inputDataSource.pos.getDetailLevel(); + + + // determine the mapping changes necessary for the input to map onto this datasource + int[] remappedIds = this.mapping.mergeAndReturnRemappedEntityIds(inputDataSource.mapping); + + boolean dataChanged = false; + if (inputDetailLevel == thisDetailLevel) + { + dataChanged = this.updateFromSameDetailLevel(inputDataSource, remappedIds); + } + else if (inputDetailLevel + 1 == thisDetailLevel) + { + dataChanged = this.updateFromOneBelowDetailLevel(inputDataSource, remappedIds); + } + else + { + // TODO what should happen here? + + // other detail levels aren't supported since it would be more difficult to maintain + // and would lead to edge cases that don't necessarily need to be supported + // (IE what do you do when the input is smaller than a single datapoint in the receiving data source?) + // instead it's better to just percolate the updates up + //throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"]."); + } + + if (dataChanged && this.pos.getDetailLevel() < NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL) + { + // mark that this data source should be applied to its parent + this.applyToParent = true; + } + } + public boolean updateFromSameDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) + { + // both data sources should have the same detail level + if (inputDataSource.pos.getDetailLevel() != this.pos.getDetailLevel()) + { + throw new IllegalArgumentException("Both data sources must have the same detail level. Expected ["+this.pos.getDetailLevel()+"], received ["+inputDataSource.pos.getDetailLevel()+"]."); + } + + // copy over everything from the input data source into this one + // provided there is data to copy and the world generation step is the same or more complete + boolean dataChanged = false; + for (int x = 0; x < WIDTH; x++) + { + for (int z = 0; z < WIDTH; z++) + { + int index = relativePosToIndex(x, z); + + long[] newDataArray = inputDataSource.dataPoints[index]; + if (newDataArray != null) + { + byte thisGenState = this.columnGenerationSteps[index]; + byte inputGenState = inputDataSource.columnGenerationSteps[index]; + + if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value + && thisGenState <= inputGenState) + { + this.dataPoints[index] = new long[newDataArray.length]; + System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); + this.remapDataColumn(index, remappedIds); + + this.columnGenerationSteps[index] = inputGenState; + + dataChanged = true; // TODO contents of the arrays should be compared to prevent re-writing the same data + } + } + } + } + + return dataChanged; + } + public boolean updateFromOneBelowDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) + { + if (inputDataSource.pos.getDetailLevel() + 1 != this.pos.getDetailLevel()) + { + throw new IllegalArgumentException("Input data source must be exactly 1 detail level below this data source. Expected ["+(this.pos.getDetailLevel() - 1)+"], received ["+inputDataSource.pos.getDetailLevel()+"]."); + } + + // input is one detail level lower (higher detail) + // so 2x2 input data points will be converted into 1 recipient data point + + + // determine where in the input data source should be written to + // since the input is one detail level below it will be one of this position's 4 children + int minChildXPos = this.pos.getChildByIndex(0).getX(); + int recipientOffsetX = (inputDataSource.pos.getX() == minChildXPos) ? 0 : (WIDTH / 2); + int minChildZPos = this.pos.getChildByIndex(0).getZ(); + int recipientOffsetZ = (inputDataSource.pos.getZ() == minChildZPos) ? 0 : (WIDTH / 2); + + + + // merge the input's data points + // into this data source's + boolean dataChanged = false; + for (int x = 0; x < WIDTH; x += 2) + { + for (int z = 0; z < WIDTH; z += 2) + { + int inputIndex = relativePosToIndex(x, z); + + long[] inputDataArray = inputDataSource.dataPoints[inputIndex]; + if (inputDataArray != null) + { + byte inputGenStep = inputDataSource.columnGenerationSteps[inputIndex]; + + // TODO downsample instad of grabbing the column nearest to (-inf, -inf) + int recipientX = (x / 2) + recipientOffsetX; + int recipientZ = (z / 2) + recipientOffsetZ; + int recipientIndex = relativePosToIndex(recipientX, recipientZ); + + this.columnGenerationSteps[recipientIndex] = inputGenStep; + this.dataPoints[recipientIndex] = inputDataArray; + this.remapDataColumn(recipientIndex, remappedIds); + + this.isEmpty = false; + + dataChanged = true; // TODO contents of the arrays should probably be compared or something + } + } + } + + return dataChanged; + } + /** + * Only update the ID once it's been added to this data source. + * Updating the incoming data source will cause issues if it is applied + * to anything else due to multiple remapping. + */ + private void remapDataColumn(int dataPointIndex, int[] remappedIds) + { + long[] dataColumn = this.dataPoints[dataPointIndex]; + for (int i = 0; i < dataColumn.length; i++) + { + dataColumn[i] = FullDataPointUtil.remap(remappedIds, dataColumn[i]); + } + } + + + + //================// + // helper methods // + //================// + + // TODO make private, any external logic should go through a method, not interact with the arrays directly + public static int relativePosToIndex(int relX, int relZ) throws IndexOutOfBoundsException + { + if (relX < 0 || relZ < 0 || + relX > WIDTH || relZ > WIDTH) + { + throw new IndexOutOfBoundsException("Relative data source positions must be between [0] and ["+WIDTH+"] (inclusive) the relative pos: ["+relX+","+relZ+"] is outside of those boundaries."); + } + + return (relX * WIDTH) + relZ; + } + + + + //=====================// + // setters and getters // + //=====================// + + @Override + public DhSectionPos getSectionPos() { return this.pos; } + + @Override + public byte getDataDetailLevel() { return (byte) (this.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); } + + @Override + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } + + @Deprecated + @Override + public EDhApiWorldGenerationStep getWorldGenStep() { return this.getWorldGenStepAtRelativePos(0, 0); } + @Override + public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) + { + int index = relativePosToIndex(relX, relZ); + return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); + } + + public boolean isEmpty() { return this.isEmpty; } + public void markNotEmpty() { this.isEmpty = false; } + + public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + { + int index = relativePosToIndex(relX, relZ); + this.dataPoints[index] = longArray; + this.columnGenerationSteps[index] = worldGenStep.value; + + + // validate the incoming ID's + // shouldn't normally happen and can be disabled for release builds + if (ModInfo.IS_DEV_BUILD) + { + int maxValidId = this.mapping.getMaxValidId(); + for (int i = 0; i < longArray.length; i++) + { + long dataPoint = longArray[i]; + int id = FullDataPointUtil.getId(dataPoint); + if (id > maxValidId) + { + LodUtil.assertNotReach("Column set with higher than possible ID. ID [" + id + "], max valid ID [" + maxValidId + "]."); + } + } + } + } + + @Override + public String toString() { return this.pos.toString(); } + + + + //============// + // deprecated // + //============// + + @Deprecated + @Override + public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) + { + throw new UnsupportedOperationException("deprecated"); + } + + + + //=========// + // pooling // + //=========// + + // TODO add pooled data sources + private static class Pooling + { + /** used when pooling data sources */ + private final ArrayList cachedSources = new ArrayList<>(); + private final ReentrantLock cacheLock = new ReentrantLock(); + + + /** @return null if no pooled source exists */ + public CompleteFullDataSource tryGetPooledSource() + { + try + { + this.cacheLock.lock(); + + int index = this.cachedSources.size() - 1; + if (index == -1) + { + return null; + } + else + { + return this.cachedSources.remove(index); + } + } + finally + { + this.cacheLock.unlock(); + } + } + + /** + * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. + * It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called. + */ + public void returnPooledDataSource(CompleteFullDataSource dataSource) + { + if (dataSource == null) + { + return; + } + else if (this.cachedSources.size() > 25) + { + return; + } + + try + { + this.cacheLock.lock(); + this.cachedSources.add(dataSource); + } + finally + { + this.cacheLock.unlock(); + } + } + + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java deleted file mode 100644 index 1dcaeed37..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; - -import com.seibel.distanthorizons.core.file.IDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.IFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import org.jetbrains.annotations.Nullable; - -import java.io.IOException; - -/** - * Base for all Full Data Source objects.

- * - * Contains full DH data, methods related to file/stream reading/writing, and the data necessary to create {@link ColumnRenderSource}'s.
- * {@link IFullDataSource}'s will either implement or contain {@link IFullDataAccessor}'s. - * - * @see IFullDataAccessor - * @see IIncompleteFullDataSource - * @see IStreamableFullDataSource - */ -public interface IFullDataSource extends IDataSource -{ - /** - * This is the byte put between different sections in the binary save file. - * The presence and absence of this byte indicates if the file is correctly formatted. - */ - int DATA_GUARD_BYTE = 0xFFFFFFFF; - /** indicates the binary save file represents an empty data source */ - int NO_DATA_FLAG_BYTE = 0x00000001; - - - - default void update(ChunkSizedFullDataAccessor chunkData, IDhLevel level) { this.update(chunkData); } - void update(ChunkSizedFullDataAccessor data); - - boolean isEmpty(); - void markNotEmpty(); - - /** AKA; the max relative position that {@link IFullDataSource#tryGet(int, int)} can accept for either X or Z */ - int getWidthInDataPoints(); - - - - //======// - // data // - //======// - - /** - * Attempts to get the data column for the given relative x and z position. - * - * @return null if the data doesn't exist - */ - @Nullable - SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ); - /** - * Attempts to get the data column for the given relative x and z position.
- * If no data exists yet an empty data column will be created. - */ - SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ); - - FullDataPointIdMap getMapping(); - - - - //=======================// - // basic stream handling // - //=======================// - - /** - * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. - * - * @see IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) - */ - void populateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; - - /** - * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. - * - * @see IStreamableFullDataSource#repopulateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) - */ - void repopulateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java deleted file mode 100644 index 4514e5f45..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; - -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.LodUtil; - -public interface IIncompleteFullDataSource extends IFullDataSource -{ - /** - * Overwrites data in this object with non-null data from the input {@link IFullDataSource}.

- * - * This can be used to either merge same sized data sources or downsample to - */ - default void sampleFrom(IFullDataSource inputSource) - { - DhSectionPos inputPos = inputSource.getSectionPos(); - DhSectionPos thisPos = this.getSectionPos(); - LodUtil.assertTrue(inputPos.getDetailLevel() < thisPos.getDetailLevel(), "input data source at pos: ["+inputPos+"] has a lower detail level than this: ["+thisPos+"]."); - LodUtil.assertTrue(inputPos.overlapsExactly(this.getSectionPos()), "input source at pos: ["+inputPos+"] (converted to ["+inputPos.convertNewToDetailLevel(thisPos.getDetailLevel())+"]) doesn't overlap with this source's pos: ["+thisPos+"]."); - - if (inputSource.isEmpty()) - { - return; - } - - - this.markNotEmpty(); - - DhLodPos baseOffset = thisPos.getMinCornerLodPos(this.getDataDetailLevel()); - DhSectionPos inputOffset = inputPos.convertNewToDetailLevel(this.getDataDetailLevel()); - int offsetX = inputOffset.getX() - baseOffset.x; - int offsetZ = inputOffset.getZ() - baseOffset.z; - - - int numberOfDataPointsToUpdate = this.getWidthInDataPoints() / thisPos.getWidthCountForLowerDetailedSection(inputSource.getSectionPos().getDetailLevel()); // can be 0 if the input source is significantly smaller than this data source - // should be 1 at minimum, to prevent divide by zero errors (and because trying to get 0 or a fractional data point doesn't make any sense) - numberOfDataPointsToUpdate = Math.max(1, numberOfDataPointsToUpdate); - - - int inputFractionWidth = inputSource.getWidthInDataPoints() / numberOfDataPointsToUpdate; - for (int x = 0; x < numberOfDataPointsToUpdate; x++) - { - for (int z = 0; z < numberOfDataPointsToUpdate; z++) - { - SingleColumnFullDataAccessor thisDataColumn = this.getOrCreate(offsetX + x, offsetZ + z); - SingleColumnFullDataAccessor inputDataColumn = inputSource.tryGet(inputFractionWidth * x, inputFractionWidth * z); - - if (inputDataColumn != null) - { - inputDataColumn.deepCopyTo(thisDataColumn); - } - } - } - } - - /** - * Attempts to convert this {@link IIncompleteFullDataSource} into a {@link CompleteFullDataSource}. - * - * @return a new {@link CompleteFullDataSource} if successful, returns itself if not. - */ - IFullDataSource tryPromotingToCompleteDataSource(); - - boolean hasBeenPromoted(); - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java deleted file mode 100644 index e1ce32bef..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; - -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; - -import java.io.IOException; - -/** - * This interface holds the complete method list necessary for reading and writing a {@link IFullDataSource} - * to and from data streams.

- * - * This interface's purpose is to reduce the chance of accidentally mismatching read/write operation data types or content by splitting - * up each read/write method into small easy to understand chunks. - * - * @param defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}. - * @param defines the object holding the data points, probably long[][] or long[][][]. - * {@link IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) populateFromStream} - * for the full reasoning. - */ -public interface IStreamableFullDataSource extends IFullDataSource -{ - - //=================// - // stream handling // - //=================// - - /** - * Clears and then overwrites any data in this object with the data from the given file and stream. - * This is expected to be used with an existing {@link IStreamableFullDataSource} and can be used in place of a constructor to reuse an existing {@link IStreamableFullDataSource} object. - * - * @see IStreamableFullDataSource#populateFromStream - */ - @Override - default void repopulateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException - { - // clear/overwrite the old data - this.resizeDataStructuresForRepopulation(dto.pos); - this.getMapping().clear(dto.pos); - - // set the new data - this.populateFromStream(dto, inputStream, level); - } - - /** - * Overwrites any data in this object with the data from the given file and stream. - * This is expected to be used with an empty {@link IStreamableFullDataSource} and functions similar to a constructor. - */ - @Override - default void populateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException - { - SummaryDataType summaryData = this.readSourceSummaryInfo(dto, inputStream, level); - this.setSourceSummaryData(summaryData); - - - DataContainerType dataPoints = this.readDataPoints(dto, summaryData.dataWidth, inputStream); - if (dataPoints == null) - { - return; - } - this.setDataPoints(dataPoints); - - - FullDataPointIdMap mapping = this.readIdMappings(dataPoints, inputStream, level.getLevelWrapper()); - this.setIdMapping(mapping); - - } - - @Override - default void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException - { - this.writeSourceSummaryInfo(level, outputStream); - - boolean hasData = this.writeDataPoints(outputStream); - if (!hasData) - { - return; - } - - this.writeIdMappings(outputStream); - } - - - - /** Note: this should only be used if the data source is being reused. Normally data sources shouldn't change. */ - void resizeDataStructuresForRepopulation(DhSectionPos pos); - - /** - * Includes information about the source file that doesn't need to be saved in each data point. Like the source's size and y-level. - */ - void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException; - /** - * Confirms that the given {@link DataSourceDto} is valid for this {@link IStreamableFullDataSource}.
- * This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed. - * - * @throws IOException if the {@link DataSourceDto} isn't valid for this object. - */ - SummaryDataType readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException; - void setSourceSummaryData(SummaryDataType summaryData); - - - /** @return true if any data points were present and written, false if this object was empty */ - boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException; - /** @return null if no data points were present */ - DataContainerType readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException; - void setDataPoints(DataContainerType dataPoints); - - - void writeIdMappings(DhDataOutputStream outputStream) throws IOException; - FullDataPointIdMap readIdMappings(DataContainerType dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException; - void setIdMapping(FullDataPointIdMap mappings); - - - - //================// - // helper classes // - //================// - - /** - * This holds information that is relevant to the entire source and isn't stored in the data points.
- * Example: minimum height, detail level, source type, etc. - */ - class FullDataSourceSummaryData - { - public final int dataWidth; - public EDhApiWorldGenerationStep worldGenStep; - - - public FullDataSourceSummaryData(int dataWidth, EDhApiWorldGenerationStep worldGenStep) - { - this.dataWidth = dataWidth; - this.worldGenStep = worldGenStep; - } - - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 2bc453a88..fdebf427e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -20,14 +20,14 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; +import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; @@ -61,8 +61,6 @@ public class ColumnRenderSource implements IDataSource public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } public static final String DATA_NAME = "ColumnRenderSource"; - @Override - public String getDataTypeName() { return DATA_NAME; } /** * This is the byte put between different sections in the binary save file. @@ -292,117 +290,60 @@ public class ColumnRenderSource implements IDataSource } @Override - public void update(ChunkSizedFullDataAccessor chunkDataView, IDhClientLevel level) + public void update(NewFullDataSource inputDataSource, IDhClientLevel level) { - final String errorMessagePrefix = "Unable to complete fastWrite for RenderSource pos: [" + this.sectionPos + "] and chunk pos: [" + chunkDataView.chunkPos + "]. Error:"; - - final DhSectionPos renderSourcePos = this.getSectionPos(); - - final int sourceBlockX = renderSourcePos.getMinCornerLodPos().getCornerBlockPos().x; - final int sourceBlockZ = renderSourcePos.getMinCornerLodPos().getCornerBlockPos().z; - - // offset between the incoming chunk data and this render source - final int blockOffsetX = (chunkDataView.chunkPos.x * LodUtil.CHUNK_WIDTH) - sourceBlockX; - final int blockOffsetZ = (chunkDataView.chunkPos.z * LodUtil.CHUNK_WIDTH) - sourceBlockZ; - - final int sourceDataPointBlockWidth = BitShiftUtil.powerOfTwo(this.getDataDetailLevel()); + final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.sectionPos + "] and pos: [" + inputDataSource.getSectionPos() + "]. Error:"; boolean dataChanged = false; - - if (chunkDataView.detailLevel == this.getDataDetailLevel()) + if (inputDataSource.getSectionPos().getDetailLevel() == this.sectionPos.getDetailLevel()) { - this.markNotEmpty(); - // confirm the render source contains this chunk - if (blockOffsetX < 0 - || blockOffsetX + LodUtil.CHUNK_WIDTH > this.getWidthInDataPoints() - || blockOffsetZ < 0 - || blockOffsetZ + LodUtil.CHUNK_WIDTH > this.getWidthInDataPoints()) + try { - LOGGER.warn(errorMessagePrefix+"Data offset is out of bounds."); - return; - } - - - if (Thread.interrupted()) - { - LOGGER.warn(errorMessagePrefix+"write interrupted."); - return; - } - - - for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++) - { - for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++) + if (Thread.interrupted()) { - ColumnArrayView columnArrayView = this.getVerticalDataPointView(blockOffsetX + x, blockOffsetZ + z); - int hash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = chunkDataView.get(x, z); - FullDataToRenderDataTransformer.convertColumnData(level, - sourceBlockX + sourceDataPointBlockWidth * (blockOffsetX + x), - sourceBlockZ + sourceDataPointBlockWidth * (blockOffsetZ + z), - columnArrayView, fullArrayView); - dataChanged |= hash != columnArrayView.getDataHash(); + LOGGER.warn(errorMessagePrefix + "write interrupted."); + return; + } + + + + DhBlockPos2D centerBlockPos = inputDataSource.getSectionPos().getCenterBlockPos(); + int halfBlockWidth = inputDataSource.getSectionPos().getBlockWidth() / 2; + DhBlockPos2D minBlockPos = new DhBlockPos2D(centerBlockPos.x - halfBlockWidth, centerBlockPos.z - halfBlockWidth); + + for (int x = 0; x < NewFullDataSource.WIDTH; x++) + { + for (int z = 0; z < NewFullDataSource.WIDTH; z++) + { + ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); + int columnHash = columnArrayView.getDataHash(); + + SingleColumnFullDataAccessor fullArrayView = inputDataSource.get(x, z); + EDhApiWorldGenerationStep worldGenStep = inputDataSource.getWorldGenStepAtRelativePos(x, z); + if (fullArrayView != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) + { + FullDataToRenderDataTransformer.convertColumnData(level, + minBlockPos.x + x, + minBlockPos.z + z, + columnArrayView, fullArrayView); + dataChanged |= columnHash != columnArrayView.getDataHash(); + + this.fillDebugFlag(x, z, 1, 1, ColumnRenderSource.DebugSourceFlag.DIRECT); + } + } } } - this.fillDebugFlag(blockOffsetX, blockOffsetZ, LodUtil.CHUNK_WIDTH, LodUtil.CHUNK_WIDTH, ColumnRenderSource.DebugSourceFlag.DIRECT); - } - else if (chunkDataView.detailLevel < this.getDataDetailLevel() && this.getDataDetailLevel() <= chunkDataView.getSectionPos().getDetailLevel()) - { - this.markNotEmpty(); - // multiple chunk data points converting to 1 column data point - DhLodPos dataCornerPos = chunkDataView.getSectionPos().getMinCornerLodPos(chunkDataView.detailLevel); - DhLodPos sourceCornerPos = renderSourcePos.getMinCornerLodPos(this.getDataDetailLevel()); - DhLodPos sourceStartingChangePos = dataCornerPos.convertToDetailLevel(this.getDataDetailLevel()); - int relStartX = Math.floorMod(sourceStartingChangePos.x, this.getWidthInDataPoints()); - int relStartZ = Math.floorMod(sourceStartingChangePos.z, this.getWidthInDataPoints()); - int dataToSourceScale = sourceCornerPos.getWidthAtDetail(chunkDataView.detailLevel); - int columnsInChunk = chunkDataView.getSectionPos().getWidthCountForLowerDetailedSection(this.getDataDetailLevel()); - - for (int xOffset = 0; xOffset < columnsInChunk; xOffset++) + catch (Exception e) { - for (int zOffset = 0; zOffset < columnsInChunk; zOffset++) - { - int relSourceX = relStartX + xOffset; - int relSourceZ = relStartZ + zOffset; - ColumnArrayView columnArrayView = this.getVerticalDataPointView(relSourceX, relSourceZ); - int hash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = chunkDataView.get(xOffset * dataToSourceScale, zOffset * dataToSourceScale); - FullDataToRenderDataTransformer.convertColumnData(level, - sourceBlockX + sourceDataPointBlockWidth * relSourceX, - sourceBlockZ + sourceDataPointBlockWidth * relSourceZ, - columnArrayView, fullArrayView); - dataChanged |= hash != columnArrayView.getDataHash(); - } + LOGGER.error(errorMessagePrefix + e.getMessage(), e); } - this.fillDebugFlag(relStartX, relStartZ, columnsInChunk, columnsInChunk, ColumnRenderSource.DebugSourceFlag.DIRECT); - } - else if (chunkDataView.getSectionPos().getDetailLevel() < this.getDataDetailLevel()) - { - // The entire chunk is being converted to a single column data point, possibly. - DhLodPos dataCornerPos = chunkDataView.getSectionPos().getMinCornerLodPos(chunkDataView.detailLevel); - DhLodPos sourceCornerPos = renderSourcePos.getMinCornerLodPos(this.getDataDetailLevel()); - DhLodPos sourceStartingChangePos = dataCornerPos.convertToDetailLevel(this.getDataDetailLevel()); - int chunksPerColumn = sourceStartingChangePos.getWidthAtDetail(chunkDataView.getSectionPos().getDetailLevel()); - if (chunkDataView.getSectionPos().getX() % chunksPerColumn != 0 || chunkDataView.getSectionPos().getZ() % chunksPerColumn != 0) - { - return; // not a multiple of the column size, so no change - } - int relStartX = Math.floorMod(sourceStartingChangePos.x, this.getWidthInDataPoints()); - int relStartZ = Math.floorMod(sourceStartingChangePos.z, this.getWidthInDataPoints()); - ColumnArrayView columnArrayView = this.getVerticalDataPointView(relStartX, relStartZ); - int hash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = chunkDataView.get(0, 0); - FullDataToRenderDataTransformer.convertColumnData(level, dataCornerPos.x * sourceDataPointBlockWidth, - dataCornerPos.z * sourceDataPointBlockWidth, - columnArrayView, fullArrayView); - dataChanged = hash != columnArrayView.getDataHash(); - this.fillDebugFlag(relStartX, relStartZ, 1, 1, ColumnRenderSource.DebugSourceFlag.DIRECT); } if (dataChanged) { this.localVersion.incrementAndGet(); + this.markNotEmpty(); } } @@ -432,7 +373,9 @@ public class ColumnRenderSource implements IDataSource public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return EDhApiWorldGenerationStep.EMPTY; } + public EDhApiWorldGenerationStep getWorldGenStep() { return EDhApiWorldGenerationStep.EMPTY; } + @Override + public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { return EDhApiWorldGenerationStep.EMPTY; } /** @return how many data points wide this {@link ColumnRenderSource} is. */ public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); } @@ -493,7 +436,6 @@ public class ColumnRenderSource implements IDataSource this.debugSourceFlags[x * SECTION_SIZE + z] = flag; } } - localVersion.incrementAndGet(); } public DebugSourceFlag debugGetFlag(int ox, int oz) { return this.debugSourceFlags[ox * SECTION_SIZE + oz]; } @@ -548,7 +490,6 @@ public class ColumnRenderSource implements IDataSource { FULL(ColorUtil.BLUE), DIRECT(ColorUtil.WHITE), - SPARSE(ColorUtil.YELLOW), FILE(ColorUtil.BROWN); public final int color; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java index 538dac67d..94caa4724 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.sql.DataSourceDto; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; @@ -31,7 +31,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; /** - * Handles loading and parsing {@link DataSourceDto}s to create {@link ColumnRenderSource}s.

+ * Handles loading and parsing {@link LegacyDataSourceDTO}s to create {@link ColumnRenderSource}s.

* * Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what * file versions this class can handle. @@ -48,7 +48,7 @@ public class ColumnRenderSourceLoader - public ColumnRenderSource loadRenderSource(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public ColumnRenderSource loadRenderSource(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataFileVersion = dto.binaryDataFormatVersion; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java index 3fd433a62..8d27273c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -23,7 +23,7 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.pos.DhChunkPos; @@ -57,7 +57,7 @@ public class ChunkToLodBuilder implements AutoCloseable // data generation // //=================// - public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) + public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) { if (chunkWrapper == null) { @@ -74,7 +74,7 @@ public class ChunkToLodBuilder implements AutoCloseable } // Otherwise, it means we're the first to do so. Let's submit our task to this entry. - CompletableFuture future = new CompletableFuture<>(); + CompletableFuture future = new CompletableFuture<>(); this.concurrentTaskToBuildList.addLast(new Task(chunkWrapper.getChunkPos(), future)); return future; } @@ -158,10 +158,10 @@ public class ChunkToLodBuilder implements AutoCloseable { if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { - ChunkSizedFullDataAccessor data = LodDataBuilder.createChunkData(latestChunk); - if (data != null) + NewFullDataSource dataSource = LodDataBuilder.createGeneratedDataSource(latestChunk); + if (dataSource != null) { - task.future.complete(data); + task.future.complete(dataSource); continue; } } @@ -233,11 +233,11 @@ public class ChunkToLodBuilder implements AutoCloseable private static class Task { public final DhChunkPos chunkPos; - public final CompletableFuture future; + public final CompletableFuture future; /** This is tracked so impossible tasks can be removed from the queue */ public long generationAttemptExpirationTimeMs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10); - Task(DhChunkPos chunkPos, CompletableFuture future) + Task(DhChunkPos chunkPos, CompletableFuture future) { this.chunkPos = chunkPos; this.future = future; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index a3b3d54d1..6c6451ecb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -22,11 +22,8 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; import com.seibel.distanthorizons.api.enums.config.EBlocksToAvoid; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; @@ -35,7 +32,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -46,8 +42,7 @@ import org.apache.logging.log4j.Logger; import java.util.HashSet; /** - * Handles converting {@link ChunkSizedFullDataAccessor}, {@link IIncompleteFullDataSource}, - * and {@link IFullDataSource}'s to {@link ColumnRenderSource}. + * Handles converting {@link NewFullDataSource}'s to {@link ColumnRenderSource}. */ public class FullDataToRenderDataTransformer { @@ -62,7 +57,7 @@ public class FullDataToRenderDataTransformer // public transformer interface // //==============================// - public static ColumnRenderSource transformFullDataToRenderSource(IFullDataSource fullDataSource, IDhClientLevel level) + public static ColumnRenderSource transformFullDataToRenderSource(NewFullDataSource fullDataSource, IDhClientLevel level) { if (fullDataSource == null) { @@ -77,17 +72,7 @@ public class FullDataToRenderDataTransformer try { - if (fullDataSource instanceof CompleteFullDataSource) - { - return transformCompleteFullDataToColumnData(level, (CompleteFullDataSource) fullDataSource); - } - else if (fullDataSource instanceof IIncompleteFullDataSource) - { - return transformIncompleteFullDataToColumnData(level, (IIncompleteFullDataSource) fullDataSource); - } - - LodUtil.assertNotReach("Unimplemented Full Data transformer for "+IFullDataSource.class.getSimpleName()+" of type ["+fullDataSource.getClass().getSimpleName()+"]."); - return null; + return transformCompleteFullDataToColumnData(level, fullDataSource); } catch (InterruptedException e) { @@ -108,7 +93,7 @@ public class FullDataToRenderDataTransformer * @throws InterruptedException Can be caused by interrupting the thread upstream. * Generally thrown if the method is running after the client leaves the current world. */ - private static ColumnRenderSource transformCompleteFullDataToColumnData(IDhClientLevel level, CompleteFullDataSource fullDataSource) throws InterruptedException + private static ColumnRenderSource transformCompleteFullDataToColumnData(IDhClientLevel level, NewFullDataSource fullDataSource) throws InterruptedException { final DhSectionPos pos = fullDataSource.getSectionPos(); final byte dataDetail = fullDataSource.getDataDetailLevel(); @@ -149,56 +134,6 @@ public class FullDataToRenderDataTransformer return columnSource; } - /** - * @throws InterruptedException Can be caused by interrupting the thread upstream. - * Generally thrown if the method is running after the client leaves the current world. - */ - private static ColumnRenderSource transformIncompleteFullDataToColumnData(IDhClientLevel level, IIncompleteFullDataSource data) throws InterruptedException - { - final DhSectionPos pos = data.getSectionPos(); - final byte dataDetail = data.getDataDetailLevel(); - final int vertSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(data.getDataDetailLevel()); - final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY()); - if (data.isEmpty()) - { - return columnSource; - } - - columnSource.markNotEmpty(); - - if (dataDetail == columnSource.getDataDetailLevel()) - { - int baseX = pos.getMinCornerLodPos().getCornerBlockPos().x; - int baseZ = pos.getMinCornerLodPos().getCornerBlockPos().z; - - int width = pos.getWidthCountForLowerDetailedSection(dataDetail); - for (int x = 0; x < width; x++) - { - for (int z = 0; z < width; z++) - { - throwIfThreadInterrupted(); - - SingleColumnFullDataAccessor fullArrayView = data.tryGet(x, z); - if (fullArrayView == null) - { - continue; - } - - ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView); - - columnSource.fillDebugFlag(x, z, 1, 1, ColumnRenderSource.DebugSourceFlag.SPARSE); - } - } - } - else - { - throw new UnsupportedOperationException("To be implemented"); - //FIXME: Implement different size creation of renderData - } - return columnSource; - } - //================// @@ -222,7 +157,9 @@ public class FullDataToRenderDataTransformer // TODO what does this mean? - private static void iterateAndConvert(IDhClientLevel level, int blockX, int blockZ, ColumnArrayView column, SingleColumnFullDataAccessor data) + private static void iterateAndConvert(IDhClientLevel level, + int blockX, int blockZ, + ColumnArrayView column, SingleColumnFullDataAccessor data) { boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); @@ -335,7 +272,7 @@ public class FullDataToRenderDataTransformer // TODO what does this mean? public static void convertColumnData(IDhClientLevel level, int blockX, int blockZ, ColumnArrayView columnArrayView, SingleColumnFullDataAccessor fullArrayView) { - if (!fullArrayView.doesColumnExist()) + if (fullArrayView == null || !fullArrayView.doesColumnExist()) { return; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 8ac100f70..65c043fda 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -21,12 +21,14 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; import java.util.List; +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.api.objects.data.DhApiChunk; import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -40,6 +42,8 @@ public class LodDataBuilder { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IBlockStateWrapper AIR = SingletonInjector.INSTANCE.get(IWrapperFactory.class).getAirBlockStateWrapper(); + /** how many chunks wide the {@link NewFullDataSource} is. */ + private static final int NUMB_OF_CHUNKS_WIDE = NewFullDataSource.WIDTH / LodUtil.CHUNK_WIDTH; private static boolean getTopErrorLogged = false; @@ -49,7 +53,7 @@ public class LodDataBuilder // converters // //============// - public static ChunkSizedFullDataAccessor createChunkData(IChunkWrapper chunkWrapper) + public static NewFullDataSource createGeneratedDataSource(IChunkWrapper chunkWrapper) { if (!canGenerateLodFromChunk(chunkWrapper)) { @@ -57,26 +61,89 @@ public class LodDataBuilder } - ChunkSizedFullDataAccessor chunkData = new ChunkSizedFullDataAccessor(chunkWrapper.getChunkPos()); - int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); - for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++) + // get the section position + int sectionPosX = chunkWrapper.getChunkPos().x; + // negative positions start at -1 so the logic there is slightly different + sectionPosX = (sectionPosX < 0) ? ((sectionPosX + 1) / NUMB_OF_CHUNKS_WIDE) - 1 : (sectionPosX / NUMB_OF_CHUNKS_WIDE); + int sectionPosZ = chunkWrapper.getChunkPos().z; + sectionPosZ = (sectionPosZ < 0) ? ((sectionPosZ + 1) / NUMB_OF_CHUNKS_WIDE) - 1 : (sectionPosZ / NUMB_OF_CHUNKS_WIDE); + DhSectionPos pos = new DhSectionPos(DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL, sectionPosX, sectionPosZ); + + NewFullDataSource dataSource = NewFullDataSource.createEmpty(pos); + dataSource.markNotEmpty(); + + + + // compute the chunk dataSource offset + // this offset is used to determine where in the dataSource this chunk's data should go + int chunkOffsetX = chunkWrapper.getChunkPos().x; + if (chunkWrapper.getChunkPos().x < 0) { - for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++) + // expected offset positions: + // chunkPos -> offset + // 5 -> 1 + // 4 -> 0 --- + // 3 -> 3 + // 2 -> 2 + // 1 -> 1 + // 0 -> 0 === + // -1 -> 3 + // -2 -> 2 + // -3 -> 1 + // -4 -> 0 --- + // -5 -> 3 + chunkOffsetX = ((chunkOffsetX) % NUMB_OF_CHUNKS_WIDE); + if (chunkOffsetX != 0) + { + chunkOffsetX += NUMB_OF_CHUNKS_WIDE; + } + } + else + { + chunkOffsetX %= NUMB_OF_CHUNKS_WIDE; + } + chunkOffsetX *= LodUtil.CHUNK_WIDTH; + + int chunkOffsetZ = chunkWrapper.getChunkPos().z; + if (chunkWrapper.getChunkPos().z < 0) + { + chunkOffsetZ = ((chunkOffsetZ) % NUMB_OF_CHUNKS_WIDE); + if (chunkOffsetZ != 0) + { + chunkOffsetZ += NUMB_OF_CHUNKS_WIDE; + } + } + else + { + chunkOffsetZ %= NUMB_OF_CHUNKS_WIDE; + } + chunkOffsetZ *= LodUtil.CHUNK_WIDTH; + + + + //==========================// + // populate the data source // + //==========================// + + int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); + for (int chunkX = 0; chunkX < LodUtil.CHUNK_WIDTH; chunkX++) + { + for (int chunkZ = 0; chunkZ < LodUtil.CHUNK_WIDTH; chunkZ++) { LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); int lastY = chunkWrapper.getMaxBuildHeight(); - IBiomeWrapper biome = chunkWrapper.getBiome(x, lastY, z); + IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); IBlockStateWrapper blockState = AIR; - int mappedId = chunkData.getMapping().addIfNotPresentAndGetId(biome, blockState); - // FIXME: The +1 offset to reproduce the old behavior. Remove this when we get per-face lighting - byte light = (byte) ((chunkWrapper.getBlockLight(x, lastY + 1, z) << 4) + chunkWrapper.getSkyLight(x, lastY + 1, z)); + int mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); + // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting + byte light = (byte) ((chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ) << 4) + chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ)); // determine the starting Y Pos - int y = chunkWrapper.getLightBlockingHeightMapValue(x,z); + int y = chunkWrapper.getLightBlockingHeightMapValue(chunkX,chunkZ); // go up until we reach open air or the world limit - IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(x, y, z); + IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight()) { try @@ -84,13 +151,13 @@ public class LodDataBuilder // This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light. // Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks. y++; - topBlockState = chunkWrapper.getBlockState(x, y, z); + topBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); } catch (Exception e) { if (!getTopErrorLogged) { - LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + x + "," + y + "," + z + "] error: " + e.getMessage(), e); + LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + chunkX + "," + y + "," + chunkZ + "] error: " + e.getMessage(), e); getTopErrorLogged = true; } @@ -102,39 +169,38 @@ public class LodDataBuilder for (; y >= minBuildHeight; y--) { - IBiomeWrapper newBiome = chunkWrapper.getBiome(x, y, z); - IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(x, y, z); - byte newLight = (byte) ((chunkWrapper.getBlockLight(x, y + 1, z) << 4) + chunkWrapper.getSkyLight(x, y + 1, z)); + IBiomeWrapper newBiome = chunkWrapper.getBiome(chunkX, y, chunkZ); + IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); + byte newLight = (byte) ((chunkWrapper.getBlockLight(chunkX, y + 1, chunkZ) << 4) + chunkWrapper.getSkyLight(chunkX, y + 1, chunkZ)); if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) { longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), light)); biome = newBiome; blockState = newBlockState; - mappedId = chunkData.getMapping().addIfNotPresentAndGetId(biome, blockState); + mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); light = newLight; lastY = y; } -// else if (newLight != light) { -// longs.add(FullFormat.encode(mappedId, lastY-y, y+1 - chunk.getMinBuildHeight(), light)); -// light = newLight; -// lastY = y; -// } } longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), light)); - chunkData.setSingleColumn(longs.toLongArray(), x, z); + dataSource.setSingleColumn(longs.toLongArray(), + chunkX + chunkOffsetX, + chunkZ + chunkOffsetZ, + EDhApiWorldGenerationStep.LIGHT); } } - if (!canGenerateLodFromChunk(chunkWrapper)) return null; - LodUtil.assertTrue(chunkData.emptyCount() == 0); - return chunkData; + + LodUtil.assertTrue(!dataSource.isEmpty()); + return dataSource; } + /** @throws ClassCastException if an API user returns the wrong object type(s) */ - public static ChunkSizedFullDataAccessor createApiChunkData(DhApiChunk dataPoints) throws ClassCastException + public static NewFullDataSource createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException { - ChunkSizedFullDataAccessor accessor = new ChunkSizedFullDataAccessor(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ)); + NewFullDataSource accessor = NewFullDataSource.createEmpty(new DhSectionPos(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ))); for (int relZ = 0; relZ < LodUtil.CHUNK_WIDTH; relZ++) { for (int relX = 0; relX < LodUtil.CHUNK_WIDTH; relX++) @@ -156,23 +222,23 @@ public class LodDataBuilder int id = accessor.getMapping().addIfNotPresentAndGetId( (IBiomeWrapper) (dataPoint.biomeWrapper), (IBlockStateWrapper) (dataPoint.blockStateWrapper) - ); + ); packedDataPoints[index] = FullDataPointUtil.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, (byte) (dataPoint.lightLevel) - ); + ); } - accessor.setSingleColumn(packedDataPoints, relX, relZ); + accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT); } } return accessor; } - + //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java similarity index 85% rename from core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index 30780d27a..5b7a4f97f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -1,11 +1,13 @@ package com.seibel.distanthorizons.core.file; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.*; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.TimerUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; @@ -29,7 +31,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; -public abstract class AbstractDataSourceHandler, TDhLevel extends IDhLevel> implements ISourceProvider +public abstract class AbstractLegacyDataSourceHandler, TDhLevel extends IDhLevel> + implements ISourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); @@ -54,7 +57,7 @@ public abstract class AbstractDataSourceHandler updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkDataView) + public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) { ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) @@ -199,11 +202,11 @@ public abstract class AbstractDataSourceHandler { - DhSectionPos bottomPos = chunkDataView.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + DhSectionPos bottomPos = inputDataSource.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); bottomPos.forEachPosUpToDetailLevel( this.topSectionDetailLevelRef.byteValue(), - (pos) -> this.updateDataSourceAtPos(pos, chunkDataView) ); + (pos) -> this.updateDataSourceAtPos(pos, inputDataSource) ); }, executor); } @@ -213,7 +216,7 @@ public abstract class AbstractDataSourceHandler
* * This prevents repeatedly reading/writing the same data source to/from disk if said @@ -279,17 +282,17 @@ public abstract class AbstractDataSourceHandler, + TDTO extends IBaseDTO, + TDhLevel extends IDhLevel> + implements ISourceProvider +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); + /** How long a data source must remain un-modified before being written to disk. */ + private static final int SAVE_DELAY_IN_MS = 4_000; + + /** + * The highest numerical detail level possible. + * Used when determining which positions to update. + * + * @see AbstractNewDataSourceHandler#MIN_SECTION_DETAIL_LEVEL + */ + public static final byte TOP_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + LodUtil.REGION_DETAIL_LEVEL; // TODO add "section" to detail level + /** + * The lowest numerical detail level possible. + * + * @see AbstractNewDataSourceHandler#TOP_SECTION_DETAIL_LEVEL + * */ + public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + + + protected final ReentrantLock[] updateLockArray; + protected final ReentrantLock closeLock = new ReentrantLock(); + protected volatile boolean isShutdown = false; + + protected final TDhLevel level; + protected final File saveDir; + + public final AbstractDhRepo repo; + + public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); + + + + //=============// + // constructor // + //=============// + + public AbstractNewDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public AbstractNewDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + { + this.level = level; + this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; + if (!this.saveDir.exists() && !this.saveDir.mkdirs()) + { + LOGGER.warn("Unable to create full data folder, file saving may fail."); + } + + // the lock array's length is double the number of CPU cores so the number of collisions + // should be relatively low without having too many extra locks + int lockCount = Runtime.getRuntime().availableProcessors() * 2; + this.updateLockArray = new ReentrantLock[lockCount]; + for (int i = 0; i < lockCount; i++) + { + this.updateLockArray[i] = new ReentrantLock(); + } + + this.repo = this.createRepo(); + } + + + + + //==================// + // abstract methods // + //==================// + + /** When this is called the parent folders should be created */ + protected abstract AbstractDhRepo createRepo(); + + protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException; + protected abstract TDTO createDtoFromDataSource(TDataSource dataSource); + + /** Creates a new data source using any DTOs already present in the database. */ + protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos); + + protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); + + + + //==============// + // data reading // + //==============// + + /** + * Returns the {@link TDataSource} for the given section position.
+ * The returned data source may be null if there was a problem.

+ * + * This call is concurrent. I.e. it supports being called by multiple threads at the same time. + */ + @Override + public CompletableFuture getAsync(DhSectionPos pos) + { + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + /** + * Should only be used in internal file handler methods where we are already running on a file handler thread. + * Can return null if there was a problem. + * @see AbstractNewDataSourceHandler#getAsync(DhSectionPos) + */ + public TDataSource get(DhSectionPos pos) + { + TDataSource dataSource = null; + try + { + TDTO dto = this.repo.getByKey(pos); + if (dto != null) + { + // load from file + dataSource = this.createDataSourceFromDto(dto); + } + else + { + // attempt to create from any existing files + dataSource = this.createNewDataSourceFromExistingDtos(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); + } + + return dataSource; + } + + + + //===============// + // data updating // + //===============// + + @Override + public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) + { + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + + try + { + // run file handling on a separate thread + return CompletableFuture.runAsync(() -> + { + this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource, true); + + }, executor); + } + catch (RejectedExecutionException ignore) + { + // can happen if the executor was shutdown while this task was queued + return CompletableFuture.completedFuture(null); + } + } + /** + * @param pos the position to update + * @param lockOnPosition Can be disabled by inheriting children to allow for their own locking logic. + * This is important if the child has its own position specific logic that shouldn't be done concurrently. + */ + protected void updateDataSourceAtPos(DhSectionPos pos, NewFullDataSource inputData, boolean lockOnPosition) + { + // a lock is necessary to prevent two threads from writing to the same position at once, + // if that happens only the second update will apply and the LOD will end up with hole(s) + ReentrantLock updateLock = this.getUpdateLockForPos(pos); + + try + { + if (lockOnPosition) + { + updateLock.lock(); + } + + + // get or create the data source + TDataSource dataSource = this.get(pos); + dataSource.update(inputData, this.level); // TODO only write to database and send update signal if data was changed + + + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(dataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + { + if (listener != null) + { + listener.OnDataSourceUpdated(dataSource); + } + } + } + catch (Exception e) + { + LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); + } + finally + { + if (lockOnPosition) + { + updateLock.unlock(); + } + } + } + + + + //================// + // helper methods // + //================// + + /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ + protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } + + + + //=========// + // cleanup // + //=========// + + @Override + public void close() + { + try + { + this.closeLock.lock(); + this.isShutdown = true; + + // wait a moment so any queued saves can finish queuing, + // otherwise we might not see everything that needs saving and attempt to use a closed repo + Thread.sleep(200); + + LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "]."); + + this.repo.close(); + } + catch (InterruptedException ignore) { } + finally + { + this.closeLock.unlock(); + } + } + + + + //================// + // helper classes // + //================// + + @FunctionalInterface + public interface IDataSourceUpdateFunc + { + void OnDataSourceUpdated(TDataSource updatedFullDataSource); + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index d1ab9424c..5a61dd2e7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -1,12 +1,11 @@ package com.seibel.distanthorizons.core.file; +import com.seibel.distanthorizons.api.enums.EDhApiDetailLevel; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.IBaseDTO; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import java.io.IOException; @@ -14,11 +13,10 @@ import java.io.IOException; /** * Base for all data sources. * - * @param what type of level this data source can be created from + * @param there are times when we need specifically a client level vs a more generic level */ public interface IDataSource extends IBaseDTO { - DhSectionPos getSectionPos(); @@ -27,8 +25,10 @@ public interface IDataSource extends IBaseDTO extends IBaseDTO - * Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse - * the binary data when read from file. + /** + * Returns the detail level of the data contained by this data source. + * IE: 0 for block, 1 for 2x2 blocks, etc. + * + * @see EDhApiDetailLevel */ - String getDataTypeName(); - /** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */ + byte getDataDetailLevel(); + + @Deprecated // TODO only necessary for full data sources + EDhApiWorldGenerationStep getWorldGenStep(); + EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ); + + /** Defines how the binary data is formatted. */ byte getDataFormatVersion(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java index 1d2b15ef4..cd9d3c363 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java @@ -1,15 +1,23 @@ package com.seibel.distanthorizons.core.file; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.util.concurrent.CompletableFuture; +/** + * Base for all data source providers + * + * @see IFullDataSourceProvider + * @see IRenderSourceProvider + */ public interface ISourceProvider, TDhLevel extends IDhLevel> extends AutoCloseable { CompletableFuture getAsync(DhSectionPos pos); - CompletableFuture updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkData); + CompletableFuture updateDataSourceAsync(NewFullDataSource inputData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java deleted file mode 100644 index 96ff63a03..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.fullDatafile; - -import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.sql.AbstractDataSourceRepo; -import com.seibel.distanthorizons.core.sql.FullDataRepo; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; - -import java.awt.*; -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; -import java.util.*; - -public class FullDataFileHandler - extends AbstractDataSourceHandler - implements IFullDataSourceProvider, IDebugRenderable -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - - - //=============// - // constructor // - //=============// - - public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } - public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) - { - super(level, saveStructure, saveDirOverride); - - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); - } - - - - //====================// - // Abstract overrides // - //====================// - - @Override - protected AbstractDataSourceRepo createRepo() - { - try - { - return new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); - } - catch (SQLException e) - { - // should only happen if there is an issue with the database (it's locked or can't be created if missing) - // or the database update failed - throw new RuntimeException(e); - } - } - - @Override - protected IFullDataSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException - { - AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.dataType, dto.binaryDataFormatVersion); - IFullDataSource dataSource = loader.loadDataSource(dto, this.level); - return dataSource; - } - /** Creates a new data source using any DTOs already present in the database. */ - @Override - protected IFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) - { - IIncompleteFullDataSource newFullDataSource = this.makeEmptyDataSource(pos); - return this.updateFromDataSourceFromExistingDtos(newFullDataSource); - } - protected IFullDataSource updateFromDataSourceFromExistingDtos(IIncompleteFullDataSource newFullDataSource) - { - DhSectionPos pos = newFullDataSource.getSectionPos(); - - boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); - if (showFullDataFileSampling) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA), - 0.2, 32f)); - } - - - // get all non-empty sections to sample from - ArrayList samplePosList = new ArrayList<>(); - ArrayList possibleChildList = new ArrayList<>(); - pos.forEachChild((childPos) -> - { - if (childPos.getDetailLevel() >= this.minDetailLevel) - { - possibleChildList.add(childPos); - } - }); - while (possibleChildList.size() != 0) - { - DhSectionPos possiblePos = possibleChildList.remove(possibleChildList.size()-1); - if (this.repo.existsWithKey(possiblePos)) - { - samplePosList.add(possiblePos); - } - else - { - possiblePos.forEachChild((childPos) -> - { - if (childPos.getDetailLevel() >= this.minDetailLevel) - { - possibleChildList.add(childPos); - } - }); - } - } - - - // read in the existing data - for (int i = 0; i < samplePosList.size(); i++) - { - DhSectionPos samplePos = samplePosList.get(i); - IFullDataSource sampleDataSource = this.get(samplePos); - if (sampleDataSource == null) - { - // no file was found, this is unexpected, but can be ignored - continue; - } - - if (showFullDataFileSampling) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA.darker()), - 0.2, 32f)); - } - - try - { - newFullDataSource.sampleFrom(sampleDataSource); - } - catch (Exception e) - { - LOGGER.warn("Unable to sample "+sampleDataSource.getSectionPos()+" into "+newFullDataSource.getSectionPos(), e); - } - } - - - // promotion may happen if all children are fully populated - return newFullDataSource.tryPromotingToCompleteDataSource(); - } - - - - @Override - protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) - { - return pos.getDetailLevel() <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL ? - HighDetailIncompleteFullDataSource.createEmpty(pos) : - LowDetailIncompleteFullDataSource.createEmpty(pos); - } - - - - //===================// - // extension methods // - //===================// - - @Override - public void writeDataSourceToFile(IFullDataSource fullDataSource) throws IOException - { - // doing this here guarantees that all changes are caught and promoted - if (fullDataSource instanceof IIncompleteFullDataSource) - { - fullDataSource = ((IIncompleteFullDataSource) fullDataSource).tryPromotingToCompleteDataSource(); - } - - super.writeDataSourceToFile(fullDataSource); - - // save has completed - boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); - if (showFullDataFileStatus) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW), - 0.2, 16f)); - } - } - - @Override - public int getUnsavedDataSourceCount() { return this.unsavedDataSourceBySectionPos.size(); } - - - - //===========// - // overrides // - //===========// - - @Override - public void debugRender(DebugRenderer renderer) - { - this.saveTimerTasksBySectionPos.keySet() - .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); - - } - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index b487b6a20..1c6e40ccd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -19,28 +19,33 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.ISourceProvider; -import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.FullDataRepo; +import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; import java.util.concurrent.CompletableFuture; /** - * Handles reading, writing, and updating {@link IFullDataSource}'s.
+ * Handles reading, writing, and updating {@link NewFullDataSource}'s.
* Should be backed by a database handled by a {@link FullDataRepo}. */ -public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable +public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable { - CompletableFuture getAsync(DhSectionPos pos); - IFullDataSource get(DhSectionPos pos); + CompletableFuture getAsync(DhSectionPos pos); + NewFullDataSource get(DhSectionPos pos); - CompletableFuture updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkData); + /** + * If this provider has the ability to create (world gen) or get (networking) + * missing data sources this method will queue the given position + * for generation or retrieval. + */ + void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos); + CompletableFuture updateDataSourceAsync(NewFullDataSource chunkData); + + @Deprecated int getUnsavedDataSourceCount(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java new file mode 100644 index 000000000..95a1e06c4 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java @@ -0,0 +1,134 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.file.fullDatafile; + +import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; +import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; +import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; + +import java.awt.*; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; + +public class LegacyFullDataFileHandler + extends AbstractLegacyDataSourceHandler + implements IDebugRenderable +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + + + //=============// + // constructor // + //=============// + + public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + { + super(level, saveStructure, saveDirOverride); + + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); + } + + + + //====================// + // Abstract overrides // + //====================// + + @Override + protected AbstractLegacyDataSourceRepo createRepo() + { + try + { + return new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + } + catch (SQLException e) + { + // should only happen if there is an issue with the database (it's locked or can't be created if missing) + // or the database update failed + throw new RuntimeException(e); + } + } + + @Override + protected CompleteFullDataSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException + { + CompleteFullDataSource dataSource = CompleteFullDataSource.createEmpty(dto.pos); + dataSource.populateFromStream(dto, dto.getInputStream(), this.level); + return dataSource; + } + /** Creates a new data source using any DTOs already present in the database. */ + @Deprecated + @Override + protected CompleteFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) + { + throw new UnsupportedOperationException("Deprecated"); + } + + + @Deprecated + @Override + protected CompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) + { + throw new UnsupportedOperationException("Deprecated"); + } + + + + //===================// + // extension methods // + //===================// + + @Deprecated + @Override + public void writeDataSourceToFile(CompleteFullDataSource fullDataSource) throws IOException + { + throw new UnsupportedOperationException("Deprecated"); + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + this.saveTimerTasksBySectionPos.keySet() + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); + + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java new file mode 100644 index 000000000..6754f1d00 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -0,0 +1,276 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.file.fullDatafile; + +import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; +import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; +import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; + +import java.awt.*; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + +public class NewFullDataFileHandler + extends AbstractNewDataSourceHandler + implements IFullDataSourceProvider, IDebugRenderable +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + private static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 20; + /** how many parent update tasks can be in the queue at once */ + private static final int MAX_PARENT_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); + + /** indicates how long the update queue thread should wait between queuing ticks */ + private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 1_000; + + + // TODO add a debug view + Set parentApplicationPositionSet = ConcurrentHashMap.newKeySet(); + private final ThreadPoolExecutor updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Update Queue Processor"); + private final AtomicBoolean updateQueueThreadRunningRef = new AtomicBoolean(false); + + + + //=============// + // constructor // + //=============// + + public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + { + super(level, saveStructure, saveDirOverride); + + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); + } + + + + //====================// + // Abstract overrides // + //====================// + + @Override + protected AbstractDhRepo createRepo() + { + try + { + return new NewFullDataSourceRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + } + catch (SQLException e) + { + // should only happen if there is an issue with the database (it's locked or the folder path is missing) + // or the database update failed + throw new RuntimeException(e); + } + } + + @Override + protected NewFullDataSourceDTO createDtoFromDataSource(NewFullDataSource dataSource) + { + try + { + return NewFullDataSourceDTO.CreateFromDataSource(dataSource); + } + catch (IOException e) + { + LOGGER.warn("Unable to create DTO, error: "+e.getMessage(), e); + return null; + } + } + + @Override + protected NewFullDataSource createDataSourceFromDto(NewFullDataSourceDTO dto) throws InterruptedException, IOException + { return dto.createDataSource(this.level.getLevelWrapper()); } + @Override + protected NewFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) + { + // TODO maybe just set children update flags to true? + return NewFullDataSource.createEmpty(pos); + } + + @Override + protected NewFullDataSource makeEmptyDataSource(DhSectionPos pos) { return NewFullDataSource.createEmpty(pos); } + + @Deprecated + @Override + public int getUnsavedDataSourceCount() { return 0; } + + + + //================// + // parent updates // + //================// + + @Override + public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) + { + // Do nothing. + // This file handler doesn't have the ability to generate or retrieve data sources + // that aren't already in the database + } + + @Override + protected void updateDataSourceAtPos(DhSectionPos pos, NewFullDataSource inputData, boolean lockOnPosition) + { + ReentrantLock updateLock = this.getUpdateLockForPos(pos); + + try + { + if (lockOnPosition) + { + updateLock.lock(); + } + + super.updateDataSourceAtPos(pos, inputData, false); + this.tryQueueParentUpdates(); + + this.parentApplicationPositionSet.remove(inputData.getSectionPos()); + if (pos.getDetailLevel() != inputData.getSectionPos().getDetailLevel()) + { + // mark that the update has completed + ((NewFullDataSourceRepo) this.repo).setApplyToParent(inputData.getSectionPos(), false); // TODO remove casting + } + } + catch (Exception e) + { + LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); + } + finally + { + if (lockOnPosition) + { + updateLock.unlock(); + } + } + } + /** Queues some of the parent updates listed in the database. */ + private void tryQueueParentUpdates() + { + // the update thread is already running, + // we don't need multiple running + if (this.updateQueueThreadRunningRef.getAndSet(true)) + { + return; + } + + + this.updateQueueProcessor.execute(() -> + { + try + { + ArrayList updatePosList = null; + + while ( // continue queuing update positions as long as there are positions to queue + (updatePosList == null || updatePosList.size() != 0) + // only add more items to the queue if half or more of the previous tasks have been completed + && this.parentApplicationPositionSet.size() < (MAX_PARENT_UPDATE_TASK_COUNT / 2)) + { + // prevent hitting the database more often than is necessary + Thread.sleep(UPDATE_QUEUE_THREAD_DELAY_IN_MS); + + + // get the positions that need to be applied to their parents + updatePosList = ((NewFullDataSourceRepo) this.repo).getPositionsToUpdate(MAX_PARENT_UPDATE_TASK_COUNT); + if (updatePosList.size() != 0) + { + // stop if the file handler has been shut down + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + this.updateQueueThreadRunningRef.set(false); + return; + } + + + // queue each update + int queueCount = 0; + for (DhSectionPos pos : updatePosList) + { + if (this.parentApplicationPositionSet.add(pos)) + { + queueCount++; + executor.execute(() -> + { + NewFullDataSource inputData = this.get(pos); + // update the parent position with this new data + this.updateDataSourceAtPos(pos.getParentPos(), inputData, true); + + // TODO add comparable interface to make this low priority + }); + } + } + + // can be used for debugging + if (queueCount != 0) + { + LOGGER.trace("Queued [" + queueCount + "] out of ["+updatePosList.size()+"] parent updates."); + } + } + } + } + catch (Exception e) + { + LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e); + } + finally + { + this.updateQueueThreadRunningRef.set(false); + } + }); + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + if (Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get()) + { + this.parentApplicationPositionSet + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); + } + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java similarity index 59% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java index 70c26ee5b..5b38ed4e8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java @@ -19,11 +19,10 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; -import com.seibel.distanthorizons.core.generation.MissingWorldGenPositionFinder; import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -34,11 +33,12 @@ import org.apache.logging.log4j.Logger; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; -public class GeneratedFullDataFileHandler extends FullDataFileHandler +public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -46,8 +46,11 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler private final ArrayList onWorldGenTaskCompleteListeners = new ArrayList<>(); - /** Used to prevent world gen tasks from being queued multiple times. */ - private final Set generatingDataPos = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + // TODO name better + // this is just the list of section pos that have had their world generation + // calculated and queued this session. + private final Set genHandledPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); @@ -55,7 +58,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // constructor // //=============// - public GeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } + public NewGeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } @@ -64,25 +67,22 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler //===========// @Override - public IFullDataSource get(DhSectionPos pos) { return this.get(pos, true); } - public IFullDataSource get(DhSectionPos pos, boolean runWorldGenCheck) + public NewFullDataSource get(DhSectionPos pos) { return this.get(pos, true); } + public NewFullDataSource get(DhSectionPos pos, boolean runWorldGenCheck) { - IFullDataSource dataSource = super.get(pos); + NewFullDataSource dataSource = super.get(pos); if (runWorldGenCheck) { - // add world gen tasks for missing columns in the data source - // if this position hasn't already been queued for generation - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null && !this.generatingDataPos.contains(pos)) - { - this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, pos, dataSource); - } + this.tryQueueSection(pos); } return dataSource; } + @Override + public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) { this.tryQueueSection(pos); } + //==================// @@ -103,17 +103,25 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public void clearGenerationQueue() { this.worldGenQueueRef.set(null); - this.generatingDataPos.clear(); // clear the incomplete data sources + this.genHandledPosSet.clear(); } /** Can be used to remove positions that are outside the player's render distance. */ public void removeGenRequestIf(Function removeIf) { - this.generatingDataPos.forEach((pos) -> + // TODO there has to be a better way to do this + + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue != null) + { + worldGenQueue.removeGenRequestIf(removeIf); + } + + this.genHandledPosSet.forEach((pos) -> { if (removeIf.apply(pos)) { - this.generatingDataPos.remove(pos); + this.genHandledPosSet.remove(pos); } }); } @@ -162,6 +170,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } } + // TODO only fire after the section has finished generated or once every X seconds private void fireOnGenPosSuccessListeners(DhSectionPos pos) { // fire the event listeners @@ -177,47 +186,97 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // helper methods // //================// - private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, DhSectionPos pos, IFullDataSource dataSource) + /** does nothing if this section or one of it's parents has already been queued */ + public void tryQueueSection(DhSectionPos pos) { - // get the un-generated pos list - byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - ArrayList genPosList = MissingWorldGenPositionFinder.getUngeneratedPosList(dataSource, minGeneratorSectionDetailLevel, true); - - // start each pos generating - ArrayList> taskFutureList = new ArrayList<>(); - for (DhSectionPos genPos : genPosList) + IWorldGenerationQueue tempWorldGenQueue = this.worldGenQueueRef.get(); + if (tempWorldGenQueue == null) { - // try not to re-queue already generating tasks - if (this.generatingDataPos.contains(genPos)) + return; + } + + if (this.genHandledPosSet.contains(pos)) + { + return; + } + + + + AtomicBoolean positionAlreadyHandled = new AtomicBoolean(false); + pos.forEachPosUpToDetailLevel(NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL, (parentPos) -> + { + if (!positionAlreadyHandled.get()) { - continue; + if (this.genHandledPosSet.contains(parentPos)) + { + positionAlreadyHandled.set(true); + } + } + }); + + if (positionAlreadyHandled.get()) + { + return; + } + this.genHandledPosSet.add(pos); + + + + // get the un-generated pos list + byte minGeneratorSectionDetailLevel = (byte) (tempWorldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + + pos.forEachChildAtDetailLevel(minGeneratorSectionDetailLevel, (genPos) -> + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return; } if (this.repo.existsWithKey(genPos)) { - continue; + // TODO only pull in the generation steps + NewFullDataSource potentialDataSource = this.get(genPos, false); + + EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT; + checkWorldGenLoop: + for (int x = 0; x < NewFullDataSource.WIDTH; x++) + { + for (int z = 0; z < NewFullDataSource.WIDTH; z++) + { + int index = NewFullDataSource.relativePosToIndex(x, z); + byte genStepValue = potentialDataSource.columnGenerationSteps[index]; + + if (genStepValue < currentMinWorldGenStep.value) + { + EDhApiWorldGenerationStep newWorldGenStep = EDhApiWorldGenerationStep.fromValue(genStepValue); + if (newWorldGenStep != null && newWorldGenStep.value < currentMinWorldGenStep.value) + { + currentMinWorldGenStep = newWorldGenStep; + } + } + + if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY) + { + // queue the task + break checkWorldGenLoop; + } + } + } + + if (currentMinWorldGenStep != EDhApiWorldGenerationStep.EMPTY) + { + // no world gen needed + return; + } } // queue each new gen task - GenTask genTask = new GenTask(dataSource.getSectionPos()); - CompletableFuture worldGenFuture = worldGenQueue.submitGenTask(genPos, dataSource.getDataDetailLevel(), genTask); + GenTask genTask = new GenTask(genPos); + CompletableFuture worldGenFuture = tempWorldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask); worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex)); - - taskFutureList.add(worldGenFuture); - } - - - // mark the data source as generating if necessary - if (taskFutureList.size() != 0) - { - this.generatingDataPos.add(pos); - CompletableFuture.allOf(taskFutureList.toArray(new CompletableFuture[0])) - .whenComplete((voidObj, ex) -> - { - this.generatingDataPos.remove(pos); - }); - } + }); } @@ -242,11 +301,11 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public boolean isMemoryAddressValid() { return true; } @Override - public Consumer getChunkDataConsumer() + public Consumer getChunkDataConsumer() { return (chunkSizedFullDataSource) -> { - GeneratedFullDataFileHandler.this.level.updateDataSourcesWithChunkData(chunkSizedFullDataSource); + NewGeneratedFullDataFileHandler.this.level.updateDataSources(chunkSizedFullDataSource); }; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewRemoteFullDataFileHandler.java similarity index 74% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewRemoteFullDataFileHandler.java index f3ef059cb..d3287db9d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewRemoteFullDataFileHandler.java @@ -25,9 +25,9 @@ import org.jetbrains.annotations.Nullable; import java.io.File; -public class RemoteFullDataFileHandler extends FullDataFileHandler +public class NewRemoteFullDataFileHandler extends NewFullDataFileHandler { - public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } - public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } + public NewRemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } + public NewRemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java index 2e31e1794..3c2147a2f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java @@ -19,14 +19,12 @@ package com.seibel.distanthorizons.core.file.renderfile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.ISourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.sql.FullDataRepo; -import com.seibel.distanthorizons.core.sql.RenderDataRepo; +import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; import java.util.concurrent.CompletableFuture; @@ -38,7 +36,7 @@ public interface IRenderSourceProvider extends ISourceProvider getAsync(DhSectionPos pos); - CompletableFuture updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkData); + CompletableFuture updateDataSourceAsync(NewFullDataSource dataSource); /** Deletes any data stored in the render cache so it can be re-created */ void deleteRenderCache(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 8b85153e4..b50dcc1d3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -19,39 +19,35 @@ package com.seibel.distanthorizons.core.file.renderfile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; +import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.sql.AbstractDataSourceRepo; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.sql.FullDataRepo; -import com.seibel.distanthorizons.core.sql.RenderDataRepo; +import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; import com.seibel.distanthorizons.core.util.threading.ThreadPools; import org.apache.logging.log4j.Logger; -import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; -public class RenderSourceFileHandler extends AbstractDataSourceHandler implements IRenderSourceProvider +public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler implements IRenderSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private final F3Screen.NestedMessage threadPoolMsg; - private final IFullDataSourceProvider fullDataSourceProvider; + public final IFullDataSourceProvider fullDataSourceProvider; @@ -78,7 +74,7 @@ public class RenderSourceFileHandler extends AbstractDataSourceHandler updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkDataView) + public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) { - return CompletableFuture.allOf( - super.updateDataSourcesWithChunkDataAsync(chunkDataView), - this.fullDataSourceProvider.updateDataSourcesWithChunkDataAsync(chunkDataView) - ); + this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); + return CompletableFuture.completedFuture(null); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 85fdb0c58..c48569e79 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -21,10 +21,9 @@ package com.seibel.distanthorizons.core.file.subDimMatching; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.structure.ClientOnlySaveStructure; import com.seibel.distanthorizons.core.generation.DhLightingEngine; @@ -190,14 +189,9 @@ public class SubDimensionLevelMatcher implements AutoCloseable LOGGER.warn("unable to build lod for chunk:"+newlyLoadedChunk.getChunkPos()); return null; } - ChunkSizedFullDataAccessor newChunkSizedFullDataView = LodDataBuilder.createChunkData(newlyLoadedChunk); - if (newChunkSizedFullDataView == null) - { - LOGGER.warn("Unexpected LOD build error for chunk:"+newlyLoadedChunk.getChunkPos()); - return null; - } + NewFullDataSource newChunkSizedFullDataView = NewFullDataSource.createFromChunk(newlyLoadedChunk); // convert to a data source for easier comparing - CompleteFullDataSource newDataSource = CompleteFullDataSource.createEmpty(new DhSectionPos(this.playerData.playerBlockPos)); + NewFullDataSource newDataSource = NewFullDataSource.createEmpty(new DhSectionPos(this.playerData.playerBlockPos)); newDataSource.update(newChunkSizedFullDataView); @@ -219,7 +213,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable { // get the data source to compare against IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), this.currentClientLevel, testLevelFolder, false); - IFullDataSource testFullDataSource = tempLevel.getFileHandler().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); + NewFullDataSource testFullDataSource = tempLevel.getFullDataProvider().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); if (testFullDataSource == null) { continue; @@ -240,8 +234,8 @@ public class SubDimensionLevelMatcher implements AutoCloseable { for (int z = 0; z < CompleteFullDataSource.WIDTH; z++) { - SingleColumnFullDataAccessor newColumn = newDataSource.tryGet(x, z); - SingleColumnFullDataAccessor testColumn = testFullDataSource.tryGet(x, z); + SingleColumnFullDataAccessor newColumn = newDataSource.get(x, z); + SingleColumnFullDataAccessor testColumn = testFullDataSource.get(x, z); if (newColumn != null && testColumn != null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java index af9c28755..52caedbef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java @@ -26,7 +26,10 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.io.Closeable; import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +// TODO does this need a interface? +@Deprecated public interface IWorldGenerationQueue extends Closeable { /** the largest numerical detail level */ @@ -45,4 +48,8 @@ public interface IWorldGenerationQueue extends Closeable CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning); void close(); + void removeGenRequestIf(Function removeIf); + void removeGenTask(DhSectionPos pos); + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/MissingWorldGenPositionFinder.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/MissingWorldGenPositionFinder.java deleted file mode 100644 index f6e554dd8..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/MissingWorldGenPositionFinder.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.generation; - -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.LinkedList; - -/** - * Handles finding any positions in a given {@link IFullDataSource} that - * aren't generated. - */ -public class MissingWorldGenPositionFinder -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - - /** - * @param generatorDetailLevel the detail level that the un-generated positions should be split into. - * @param onlyReturnPositionsTheGeneratorCanAccept - * If true this will only return positions with the same detail level as generatorDetailLevel.
- * If false this will return the lowest detail level possible for totally empty sections.
- * As of 2023-9-28 both have been tested and confirmed working with the Batch world generator, the only difference is that - * the task list will be deceptively small if this value is false. - * - * @return the list of {@link DhSectionPos} that aren't generated in this data source. - */ - public static ArrayList getUngeneratedPosList(IFullDataSource dataSource, byte generatorDetailLevel, boolean onlyReturnPositionsTheGeneratorCanAccept) - { - ArrayList posArray = getUngeneratedPosListForQuadrant(dataSource, dataSource.getSectionPos(), generatorDetailLevel); - - if (onlyReturnPositionsTheGeneratorCanAccept) - { - LinkedList posList = new LinkedList<>(posArray); - - // subdivide positions until they match the generatorDetailLevel - ArrayList cleanedPosArray = new ArrayList<>(); - while (posList.size() > 0) - { - DhSectionPos pos = posList.remove(); - if (pos.getDetailLevel() > generatorDetailLevel) - { - pos.forEachChild((childPos) -> posList.push(childPos)); - } - else - { - cleanedPosArray.add(pos); - } - } - - return cleanedPosArray; - } - else - { - return posArray; - } - } - private static ArrayList getUngeneratedPosListForQuadrant(IFullDataSource dataSource, DhSectionPos quadrantPos, byte generatorDetailLevel) - { - ArrayList ungeneratedPosList = new ArrayList<>(); - - int sourceRelWidthInDataPoints = dataSource.getWidthInDataPoints(); - - - if (quadrantPos.getDetailLevel() == generatorDetailLevel) - { - // we are at the highest detail level the world generator can accept, - // we either need to generate this whole section, or not at all - - ESectionPopulationState populationState = getPopulationStateForPos(dataSource, quadrantPos, sourceRelWidthInDataPoints); - if (populationState != ESectionPopulationState.COMPLETE) - { - // at least 1 data point is missing, - // this whole section must be generated - ungeneratedPosList.add(quadrantPos); - } - } - else if (quadrantPos.getDetailLevel() > generatorDetailLevel) - { - // detail level too low for world generator, check child positions - for (int i = 0; i < 4; i++) - { - DhSectionPos inputPos = quadrantPos.getChildByIndex(i); - - ESectionPopulationState populationState = getPopulationStateForPos(dataSource, inputPos, sourceRelWidthInDataPoints); - if (populationState == ESectionPopulationState.COMPLETE) - { - // no generation necessary - continue; - } - else if (populationState == ESectionPopulationState.EMPTY) - { - // nothing exists for this sub quadrant, add this sub-quadrant's position - ungeneratedPosList.add(inputPos); - } - else if (populationState == ESectionPopulationState.PARTIAL) - { - // some data exists in this quadrant, but not all that we need - // recurse down to determine which sub-quadrant positions will need generation - ungeneratedPosList.addAll(getUngeneratedPosListForQuadrant(dataSource, inputPos, generatorDetailLevel)); - } - } - } - else - { - throw new IllegalArgumentException("detail level lower than world generator can accept."); - } - - return ungeneratedPosList; - } - private static ESectionPopulationState getPopulationStateForPos(IFullDataSource dataSource, DhSectionPos inputPos, int sourceRelWidthInDataPoints) - { - // TODO comment - - byte childDetailLevel = inputPos.getDetailLevel(); - - int quadrantDetailLevelDiff = dataSource.getSectionPos().getDetailLevel() - childDetailLevel; - int widthInSecPos = BitShiftUtil.powerOfTwo(quadrantDetailLevelDiff); - int relWidthForSecPos = sourceRelWidthInDataPoints / widthInSecPos; - - DhSectionPos minSecPos = dataSource.getSectionPos().convertNewToDetailLevel(childDetailLevel); - - - - int minRelX = inputPos.getX() - minSecPos.getX(); - int maxRelX = minRelX + 1; - - int minRelZ = inputPos.getZ() - minSecPos.getZ(); - int maxRelZ = minRelZ + 1; - - minRelX = minRelX * relWidthForSecPos; - maxRelX = maxRelX * relWidthForSecPos; - - minRelZ = minRelZ * relWidthForSecPos; - maxRelZ = maxRelZ * relWidthForSecPos; - - - - boolean quadrantFullyGenerated = true; - boolean quadrantEmpty = true; - for (int relX = minRelX; relX < maxRelX; relX++) - { - for (int relZ = minRelZ; relZ < maxRelZ; relZ++) - { - SingleColumnFullDataAccessor column = dataSource.tryGet(relX, relZ); - if (column == null || !column.doesColumnExist()) - { - // no data for this relative position - quadrantFullyGenerated = false; - } - else - { - // data exists for this pos - quadrantEmpty = false; - } - } - } - - - if (quadrantFullyGenerated) - { - // no generation necessary - return ESectionPopulationState.COMPLETE; - } - else if (quadrantEmpty) - { - // nothing exists for this sub quadrant, add this sub-quadrant's position - return ESectionPopulationState.EMPTY; - } - else - { - // some data exists in this quadrant, but not all that we need - // recurse down to determine which sub-quadrant positions will need generation - return ESectionPopulationState.PARTIAL; - } - } - - - - //================// - // helper classes // - //================// - - private enum ESectionPopulationState - { - COMPLETE, - EMPTY, - PARTIAL - } -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 7c3b68744..822a86606 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiDistantGenerat import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratorReturnType; import com.seibel.distanthorizons.api.objects.data.DhApiChunk; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.generation.tasks.*; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -47,6 +47,7 @@ import java.awt.*; import java.util.*; import java.util.concurrent.*; import java.util.function.Consumer; +import java.util.function.Function; public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRenderable { @@ -159,6 +160,29 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender return future; } + @Override + public void removeGenRequestIf(Function removeIf) + { + this.waitingTasks.forEachKey(100, (genPos) -> + { + if (removeIf.apply(genPos)) + { + this.waitingTasks.remove(genPos); + } + }); + } + + @Override + public void removeGenTask(DhSectionPos pos) + { + WorldGenTask task = this.waitingTasks.remove(pos); + if (task != null) + { + task.future.cancel(true); + } + } + + //===============// @@ -407,7 +431,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, - Consumer chunkDataConsumer + Consumer chunkDataConsumer ) { EDhApiDistantGeneratorMode generatorMode = Config.Client.Advanced.WorldGenerator.distantGeneratorMode.get(); @@ -428,9 +452,9 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender try { IChunkWrapper chunk = WRAPPER_FACTORY.createChunkWrapper(generatedObjectArray); - ChunkSizedFullDataAccessor chunkDataAccessor = LodDataBuilder.createChunkData(chunk); - LodUtil.assertTrue(chunkDataAccessor != null); - chunkDataConsumer.accept(chunkDataAccessor); + NewFullDataSource dataSource = LodDataBuilder.createGeneratedDataSource(chunk); + LodUtil.assertTrue(dataSource != null); + chunkDataConsumer.accept(dataSource); } catch (ClassCastException e) { @@ -453,8 +477,8 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender { try { - ChunkSizedFullDataAccessor chunkDataAccessor = LodDataBuilder.createApiChunkData(dataPoints); - chunkDataConsumer.accept(chunkDataAccessor); + NewFullDataSource dataSource = LodDataBuilder.createFromApiChunkData(dataPoints); + chunkDataConsumer.accept(dataSource); } catch (ClassCastException e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java index d30653121..4568f9821 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.generation.tasks; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import java.util.function.Consumer; @@ -32,6 +32,6 @@ public interface IWorldGenTaskTracker /** Returns true if the task hasn't been garbage collected. */ boolean isMemoryAddressValid(); - Consumer getChunkDataConsumer(); + Consumer getChunkDataConsumer(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java index fdc188d55..39c145c01 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java @@ -19,8 +19,7 @@ package com.seibel.distanthorizons.core.generation.tasks; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.pos.DhLodPos; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.util.Iterator; @@ -31,6 +30,7 @@ import java.util.function.Consumer; * @author Leetom * @version 2022-11-25 */ +@Deprecated // TODO look into how these are used and if they should continue to be used public final class WorldGenTaskGroup { public final DhSectionPos pos; @@ -46,13 +46,13 @@ public final class WorldGenTaskGroup this.dataDetail = dataDetail; } - public void consumeChunkData(ChunkSizedFullDataAccessor chunkSizedFullDataView) + public void consumeChunkData(NewFullDataSource chunkSizedFullDataView) { Iterator tasks = this.worldGenTasks.iterator(); while (tasks.hasNext()) { WorldGenTask task = tasks.next(); - Consumer chunkDataConsumer = task.taskTracker.getChunkDataConsumer(); + Consumer chunkDataConsumer = task.taskTracker.getChunkDataConsumer(); if (chunkDataConsumer == null) { tasks.remove(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java index 72e3324c7..24e76ba78 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java @@ -20,13 +20,11 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiChunkModifiedEvent; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.transformers.ChunkToLodBuilder; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector; -import java.util.concurrent.CompletableFuture; - public abstract class AbstractDhLevel implements IDhLevel { public final ChunkToLodBuilder chunkToLodBuilder; @@ -48,23 +46,17 @@ public abstract class AbstractDhLevel implements IDhLevel @Override public void updateChunkAsync(IChunkWrapper chunk) { - CompletableFuture future = this.chunkToLodBuilder.tryGenerateData(chunk); - if (future != null) + NewFullDataSource dataSource = NewFullDataSource.createFromChunk(chunk); + if (dataSource == null) { - future.thenAccept((chunkSizedFullDataAccessor) -> - { - if (chunkSizedFullDataAccessor == null) - { - // This can happen if, among other reasons, a chunk save is superceded by a later event - return; - } - - this.updateDataSourcesWithChunkData(chunkSizedFullDataAccessor); - ApiEventInjector.INSTANCE.fireAllEvents( - DhApiChunkModifiedEvent.class, - new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunk.getChunkPos().x, chunk.getChunkPos().z)); - }); + // This can happen if, among other reasons, a chunk save is superseded by a later event + return; } + + this.updateDataSources(dataSource); + ApiEventInjector.INSTANCE.fireAllEvents( + DhApiChunkModifiedEvent.class, + new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunk.getChunkPos().x, chunk.getChunkPos().z)); } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index e83aa706f..23a358d7e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -22,9 +22,11 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; +import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -38,25 +40,36 @@ import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; -import com.seibel.distanthorizons.coreapi.util.math.Mat4f; import org.apache.logging.log4j.Logger; import java.io.Closeable; import java.util.concurrent.atomic.AtomicReference; -public class ClientLevelModule implements Closeable +public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); private final IDhClientLevel parentClientLevel; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); public final F3Screen.NestedMessage f3Message; + + + + //=============// + // constructor // + //=============// + public ClientLevelModule(IDhClientLevel parentClientLevel) { this.parentClientLevel = parentClientLevel; this.f3Message = new F3Screen.NestedMessage(this::f3Log); + + NewFullDataFileHandler fileHandler = this.parentClientLevel.getFullDataProvider(); + fileHandler.dateSourceUpdateListeners.add(this); } + + //==============// // tick methods // //==============// @@ -92,7 +105,7 @@ public class ClientLevelModule implements Closeable } clientRenderState.close(); - clientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFileHandler(), this.parentClientLevel.getSaveStructure()); + clientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, clientRenderState)) { //FIXME: How to handle this? @@ -124,7 +137,7 @@ public class ClientLevelModule implements Closeable /** @return if the {@link ClientRenderState} was successfully swapped */ public boolean startRenderer(IClientLevelWrapper clientLevelWrapper) { - ClientRenderState ClientRenderState = new ClientRenderState(parentClientLevel, clientLevelWrapper, parentClientLevel.getFileHandler(), parentClientLevel.getSaveStructure()); + ClientRenderState ClientRenderState = new ClientRenderState(parentClientLevel, clientLevelWrapper, parentClientLevel.getFullDataProvider(), parentClientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, ClientRenderState)) { LOGGER.warn("Failed to start renderer due to concurrency"); @@ -185,22 +198,23 @@ public class ClientLevelModule implements Closeable ClientRenderState.close(); } + + //===============// // data handling // //===============// - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) + + public void updateDataSources(NewFullDataSource data) { this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } + @Override + public void OnDataSourceUpdated(NewFullDataSource updatedFullDataSource) { ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { ClientRenderState.renderSourceFileHandler - .updateDataSourcesWithChunkDataAsync(data) + .updateDataSourceAsync(updatedFullDataSource) // wait for the update to finish before triggering a reload to prevent holes in the world - .thenRun(() -> ClientRenderState.quadtree.reloadPos(data.sectionPos)); - } - else - { - this.parentClientLevel.getFileHandler().updateDataSourcesWithChunkDataAsync(data); + .thenRun(() -> ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos())); } } @@ -231,16 +245,10 @@ public class ClientLevelModule implements Closeable - //=======================// // misc helper functions // //=======================// - public void dumpRamUsage() - { - //TODO - } - /** Returns what should be displayed in Minecraft's F3 debug menu */ protected String[] f3Log() { @@ -274,6 +282,12 @@ public class ClientLevelModule implements Closeable } } + + + //================// + // helper classes // + //================// + public static class ClientRenderState { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 1c69e5c83..500588d13 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -20,9 +20,10 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewRemoteFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; @@ -44,7 +45,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public final ClientLevelModule clientside; public final IClientLevelWrapper levelWrapper; public final AbstractSaveStructure saveStructure; - public final RemoteFullDataFileHandler dataFileHandler; + public final NewRemoteFullDataFileHandler dataFileHandler; @@ -57,7 +58,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel { this.levelWrapper = clientLevelWrapper; this.saveStructure = saveStructure; - this.dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure, fullDataSaveDirOverride); + this.dataFileHandler = new NewRemoteFullDataFileHandler(this, saveStructure, fullDataSaveDirOverride); this.clientside = new ClientLevelModule(this); if (enableRendering) @@ -117,7 +118,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public ILevelWrapper getLevelWrapper() { return levelWrapper; } @Override - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); } + public void updateDataSources(NewFullDataSource data) { this.clientside.updateDataSources(data); } @Override public int getMinY() { return levelWrapper.getMinHeight(); } @@ -136,10 +137,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel //=======================// @Override - public IFullDataSourceProvider getFileHandler() - { - return dataFileHandler; - } + public NewFullDataFileHandler getFullDataProvider() { return this.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index be70e9f95..8a063f933 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -20,9 +20,10 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.render.LodRenderSection; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; @@ -173,10 +174,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public IFullDataSourceProvider getFileHandler() - { - return serverside.dataFileHandler; - } + public NewFullDataFileHandler getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() @@ -188,7 +186,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public boolean hasSkyLight() { return this.serverLevelWrapper.hasSkyLight(); } @Override - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); } + public void updateDataSources(NewFullDataSource data) { this.clientside.updateDataSources(data); } @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index 9b12a3627..e8cdd66de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -50,11 +50,11 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public void serverTick() { this.chunkToLodBuilder.tick(); } @Override - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) + public void updateDataSources(NewFullDataSource data) { DhSectionPos pos = data.getSectionPos(); pos = pos.convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); - this.getFileHandler().updateDataSourcesWithChunkDataAsync(data); + this.getFullDataProvider().updateDataSourceAsync(data); } @Override @@ -97,7 +97,7 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public IFullDataSourceProvider getFileHandler() { return serverside.dataFileHandler; } + public NewFullDataFileHandler getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java index f614dad86..02331409b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java @@ -19,14 +19,12 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import java.util.concurrent.CompletableFuture; - public interface IDhLevel extends AutoCloseable { int getMinY(); @@ -39,12 +37,12 @@ public interface IDhLevel extends AutoCloseable void updateChunkAsync(IChunkWrapper chunk); - IFullDataSourceProvider getFileHandler(); + NewFullDataFileHandler getFullDataProvider(); AbstractSaveStructure getSaveStructure(); boolean hasSkyLight(); - void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data); + void updateDataSources(NewFullDataSource data); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java index 94108b380..118d4a272 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; -public interface IDhWorldGenLevel extends IDhLevel, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener +public interface IDhWorldGenLevel extends IDhLevel, NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener { void doWorldGen(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java index e73deca38..cdc32f285 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java @@ -22,12 +22,11 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator; import com.seibel.distanthorizons.core.config.AppliedConfigState; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.BatchGenerator; import com.seibel.distanthorizons.core.generation.WorldGenerationQueue; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; import com.seibel.distanthorizons.coreapi.DependencyInjection.WorldGeneratorInjector; import org.apache.logging.log4j.Logger; @@ -37,7 +36,7 @@ public class ServerLevelModule public final IDhServerLevel parentServerLevel; public final AbstractSaveStructure saveStructure; - public final GeneratedFullDataFileHandler dataFileHandler; + public final NewGeneratedFullDataFileHandler dataFileHandler; public final AppliedConfigState worldGeneratorEnabledConfig; public final WorldGenModule worldGenModule; @@ -48,7 +47,7 @@ public class ServerLevelModule { this.parentServerLevel = parentServerLevel; this.saveStructure = saveStructure; - this.dataFileHandler = new GeneratedFullDataFileHandler(parentServerLevel, saveStructure); + this.dataFileHandler = new NewGeneratedFullDataFileHandler(parentServerLevel, saveStructure); this.worldGeneratorEnabledConfig = new AppliedConfigState<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration); this.worldGenModule = new WorldGenModule(this.dataFileHandler, this.parentServerLevel); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index 5b274857f..f008660b1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -34,15 +34,15 @@ public class WorldGenModule implements Closeable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final GeneratedFullDataFileHandler dataFileHandler; - private final GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener; + private final NewGeneratedFullDataFileHandler dataFileHandler; + private final NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener; private final AtomicReference worldGenStateRef = new AtomicReference<>(); private final F3Screen.DynamicMessage worldGenF3Message; - public WorldGenModule(GeneratedFullDataFileHandler dataFileHandler, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener) + public WorldGenModule(NewGeneratedFullDataFileHandler dataFileHandler, NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener) { this.dataFileHandler = dataFileHandler; this.onWorldGenCompleteListener = onWorldGenCompleteListener; @@ -70,7 +70,7 @@ public class WorldGenModule implements Closeable // world gen control // //===================// - public void startWorldGen(GeneratedFullDataFileHandler dataFileHandler, AbstractWorldGenState newWgs) + public void startWorldGen(NewGeneratedFullDataFileHandler dataFileHandler, AbstractWorldGenState newWgs) { // create the new world generator if (!this.worldGenStateRef.compareAndSet(null, newWgs)) @@ -82,7 +82,7 @@ public class WorldGenModule implements Closeable dataFileHandler.setWorldGenerationQueue(newWgs.worldGenerationQueue); } - public void stopWorldGen(GeneratedFullDataFileHandler dataFileHandler) + public void stopWorldGen(NewGeneratedFullDataFileHandler dataFileHandler) { AbstractWorldGenState worldGenState = this.worldGenStateRef.get(); if (worldGenState == null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java index d6d038b74..b2f3cb1f4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.pos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.LodUtil; import org.jetbrains.annotations.NotNull; @@ -32,6 +32,7 @@ import java.util.Objects; * @author Leetom * @version 2022-11-6 */ +@Deprecated public class DhLodPos implements Comparable { public final byte detailLevel; @@ -116,7 +117,7 @@ public class DhLodPos implements Comparable public DhLodPos getDhSectionRelativePositionForDetailLevel() throws IllegalArgumentException { return this.getDhSectionRelativePositionForDetailLevel(this.detailLevel); } /** * Returns a DhLodPos with the given detail level and an X/Z position somewhere between (0,0) and (63,63). - * This is done to access specific sections from a {@link IFullDataSource} where LOD columns are stored + * This is done to access specific sections from a {@link NewFullDataSource} where LOD columns are stored * in 64 x 64 blocks. * * @throws IllegalArgumentException if this position's detail level is lower than the output detail level diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index 819e2caa8..c6cc3a0ab 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -155,8 +155,10 @@ public class DhSectionPos //=========// /** @return the corner with the smallest X and Z coordinate */ + @Deprecated public DhLodPos getMinCornerLodPos() { return this.getMinCornerLodPos((byte) (this.detailLevel - 1)); } /** @return the corner with the smallest X and Z coordinate */ + @Deprecated public DhLodPos getMinCornerLodPos(byte returnDetailLevel) { LodUtil.assertTrue(returnDetailLevel <= this.detailLevel, "returnDetailLevel must be less than sectionDetail"); @@ -167,6 +169,16 @@ public class DhSectionPos this.z * BitShiftUtil.powerOfTwo(offset)); } + public DhSectionPos getMinCornerPos(byte returnDetailLevel) + { + LodUtil.assertTrue(returnDetailLevel <= this.detailLevel, "returnDetailLevel must be less than sectionDetail"); + + byte offset = (byte) (this.detailLevel - returnDetailLevel); + return new DhSectionPos(returnDetailLevel, + this.x * BitShiftUtil.powerOfTwo(offset), + this.z * BitShiftUtil.powerOfTwo(offset)); + } + /** * A detail level of X lower than this section's detail level will return:
* 0 -> 1
@@ -331,7 +343,7 @@ public class DhSectionPos } /** Applies the given consumer to all children of the position at the given section detail level. */ - public void forEachChildAtLevel(byte sectionDetailLevel, Consumer callback) + public void forEachChildAtDetailLevel(byte sectionDetailLevel, Consumer callback) { if (sectionDetailLevel == this.detailLevel) { @@ -341,7 +353,7 @@ public class DhSectionPos for (int i = 0; i < 4; i++) { - this.getChildByIndex(i).forEachChildAtLevel(sectionDetailLevel, callback); + this.getChildByIndex(i).forEachChildAtDetailLevel(sectionDetailLevel, callback); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 2f9a9bda9..5ed35e536 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -33,6 +33,7 @@ import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree; import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import java.util.Iterator; import java.util.concurrent.ConcurrentLinkedQueue; @@ -206,7 +207,7 @@ public class LodQuadTree extends QuadTree implements AutoClose LodRenderSection newRenderSection = new LodRenderSection(this, sectionPos); rootNode.setValue(sectionPos, newRenderSection); - renderSection = newRenderSection; + renderSection = newRenderSection; // TODO this never seemed to be called, is it necessary? } @@ -435,7 +436,7 @@ public class LodQuadTree extends QuadTree implements AutoClose * Can be called whenever a render section's data needs to be refreshed.
* This should be called whenever a world generation task is completed or if the connected server has new data to show. */ - public void reloadPos(DhSectionPos pos) + public void reloadPos(@NotNull DhSectionPos pos) { if (pos == null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java index d71eda424..2c4c56844 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -20,6 +20,8 @@ package com.seibel.distanthorizons.core.sql; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import org.apache.logging.log4j.Logger; import java.io.IOException; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java similarity index 95% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java index a692a5b08..36405f26b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java @@ -17,7 +17,7 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.dto; /** * DTO = DataTable Object
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java similarity index 83% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java index a0782c41b..2f7ec8de0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java @@ -17,10 +17,9 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.dto; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -30,8 +29,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; -/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */ -public class DataSourceDto implements IBaseDTO +/** handles storing {@link ColumnRenderSource}'s in the database. */ +public class LegacyDataSourceDTO implements IBaseDTO { public DhSectionPos pos; public int checksum; @@ -54,7 +53,7 @@ public class DataSourceDto implements IBaseDTO // constructor // //=============// - public DataSourceDto(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) + public LegacyDataSourceDTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) { this.pos = pos; this.checksum = checksum; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java new file mode 100644 index 000000000..154b47230 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java @@ -0,0 +1,256 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.sql.dto; + +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import org.jetbrains.annotations.NotNull; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.Adler32; +import java.util.zip.CheckedOutputStream; + +/** handles storing {@link NewFullDataSource}'s in the database. */ +public class NewFullDataSourceDTO implements IBaseDTO +{ + public DhSectionPos pos; + + public int levelMinY; + + /** only for the data array */ + public int dataChecksum; + + //public long[][] dataArray; + public byte[] dataByteArray; + + /** @see EDhApiWorldGenerationStep */ + public byte[] columnGenStepByteArray; + + //public FullDataPointIdMap mapping; + public byte[] mappingByteArray; + + public byte dataFormatVersion; + + public boolean applyToParent; + + public long lastModifiedUnixDateTime; + public long createdUnixDateTime; + + + + //=============// + // constructor // + //=============// + + public static NewFullDataSourceDTO CreateFromDataSource(NewFullDataSource dataSource) throws IOException + { + CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints); + byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping()); + + return new NewFullDataSourceDTO( + dataSource.getSectionPos(), + checkedDataPointArray.checksum, dataSource.columnGenerationSteps, NewFullDataSource.DATA_FORMAT_VERSION, checkedDataPointArray.byteArray, + dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, + mappingByteArray, dataSource.applyToParent, + dataSource.levelMinY + ); + } + public NewFullDataSourceDTO( + DhSectionPos pos, + int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, byte[] dataByteArray, + long lastModifiedUnixDateTime, long createdUnixDateTime, + byte[] mappingByteArray, boolean applyToParent, + int levelMinY) + { + this.pos = pos; + this.dataChecksum = dataChecksum; + this.columnGenStepByteArray = columnGenStepByteArray; + + this.dataFormatVersion = dataFormatVersion; + + this.dataByteArray = dataByteArray; + this.mappingByteArray = mappingByteArray; + + this.applyToParent = applyToParent; + + this.lastModifiedUnixDateTime = lastModifiedUnixDateTime; + this.createdUnixDateTime = createdUnixDateTime; + + this.levelMinY = levelMinY; + } + + + + //========================// + // data source population // + //========================// + + public NewFullDataSource createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { return this.populateDataSource(NewFullDataSource.createEmpty(this.pos), levelWrapper); } + public NewFullDataSource populateDataSource(NewFullDataSource dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { + if (NewFullDataSource.DATA_FORMAT_VERSION != this.dataFormatVersion) + { + throw new IllegalStateException("There should only be one data format right now anyway."); + } + + dataSource.columnGenerationSteps = this.columnGenStepByteArray; + dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray); + + dataSource.getMapping().clear(dataSource.getSectionPos()); + dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper)); + + dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; + dataSource.createdUnixDateTime = this.createdUnixDateTime; + + dataSource.levelMinY = this.levelMinY; + + return dataSource; + } + + + + //=================// + // (de)serializing // + //=================// + + private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray) throws IOException + { + // write the outputs to a stream to prep for writing to the database + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + + // the order of these streams is important, otherwise the checksum won't be calculated + CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); + // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, + // but since this stream will be closed immediately after writing anyway, it won't be an issue + DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut); + + + // write the data + int dataArrayLength = NewFullDataSource.WIDTH * NewFullDataSource.WIDTH; + for (int xz = 0; xz < dataArrayLength; xz++) + { + long[] dataColumn = dataArray[xz]; + + // write column length + int columnLength = (dataColumn != null) ? dataColumn.length : 0; + compressedOut.writeInt(columnLength); + + // write column data (will be skipped if no data was present) + for (int y = 0; y < columnLength; y++) + { + compressedOut.writeLong(dataColumn[y]); + } + } + + + // generate the checksum + compressedOut.flush(); + int checksum = (int) checkedOut.getChecksum().getValue(); + byteArrayOutputStream.close(); + + return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); + } + private static long[][] readBlobToDataSourceDataArray(byte[] dataByteArray) throws IOException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream); + + + // read the data + int dataArrayLength = NewFullDataSource.WIDTH * NewFullDataSource.WIDTH; + long[][] dataArray = new long[dataArrayLength][]; + for (int xz = 0; xz < dataArray.length; xz++) + { + // read the column length + int dataColumnLength = compressedIn.readInt(); // separate variables are used for debugging and in case validation wants to be added later + long[] dataColumn = new long[dataColumnLength]; + + // read column data (will be skipped if no data was present) + for (int y = 0; y < dataColumnLength; y++) + { + long dataPoint = compressedIn.readLong(); + dataColumn[y] = dataPoint; + } + + dataArray[xz] = dataColumn; + } + + + return dataArray; + } + + + private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping) throws IOException + { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream); + + mapping.serialize(compressedOut); + + compressedOut.flush(); + byteArrayOutputStream.close(); + + return byteArrayOutputStream.toByteArray(); + } + private static FullDataPointIdMap readBlobToDataMapping(byte[] dataByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream); + + FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper); + return mapping; + } + + + + //===========// + // overrides // + //===========// + + @Override + public DhSectionPos getKey() { return this.pos; } + + + + //================// + // helper classes // + //================// + + private static class CheckedByteArray + { + public final int checksum; + public final byte[] byteArray; + + public CheckedByteArray(int checksum, byte[] byteArray) + { + this.checksum = checksum; + this.byteArray = byteArray; + } + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java similarity index 98% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index 4acbe209d..2420a0478 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -17,9 +17,12 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.sql.DatabaseUpdater; +import com.seibel.distanthorizons.core.sql.DbConnectionClosedException; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java similarity index 85% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java index a6d94a053..bc979094c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java @@ -17,20 +17,21 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Map; -public abstract class AbstractDataSourceRepo extends AbstractDhRepo +public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo { - public AbstractDataSourceRepo(String databaseType, String databaseLocation) throws SQLException + public AbstractLegacyDataSourceRepo(String databaseType, String databaseLocation) throws SQLException { - super(databaseType, databaseLocation, DataSourceDto.class); + super(databaseType, databaseLocation, LegacyDataSourceDTO.class); } @@ -40,7 +41,7 @@ public abstract class AbstractDataSourceRepo extends AbstractDhRepo objectMap) throws ClassCastException + public LegacyDataSourceDTO convertDictionaryToDto(Map objectMap) throws ClassCastException { String posString = (String) objectMap.get("DhSectionPos"); DhSectionPos pos = DhSectionPos.deserialize(posString); @@ -58,7 +59,7 @@ public abstract class AbstractDataSourceRepo extends AbstractDhRepo. */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.sql.SQLException; -public class FullDataRepo extends AbstractDataSourceRepo +public class FullDataRepo extends AbstractLegacyDataSourceRepo { public static final String TABLE_NAME = "DhFullData"; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java new file mode 100644 index 000000000..6f0ab5ff0 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java @@ -0,0 +1,214 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.sql.repo; + +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class NewFullDataSourceRepo extends AbstractDhRepo +{ + public NewFullDataSourceRepo(String databaseType, String databaseLocation) throws SQLException + { + super(databaseType, databaseLocation, NewFullDataSourceDTO.class); + } + + + + @Override + public String getTableName() { return "FullData"; } + + @Override + public String createWhereStatement(DhSectionPos pos) + { + int detailLevel = pos.getDetailLevel() - DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL; + return "DetailLevel = '"+detailLevel+"' AND PosX = '"+pos.getX()+"' AND PosZ = '"+pos.getZ()+"'"; + } + + + + //=======================// + // repo required methods // + //=======================// + + @Override + public NewFullDataSourceDTO convertDictionaryToDto(Map objectMap) throws ClassCastException + { + byte detailLevel = (Byte) objectMap.get("DetailLevel"); + byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + int posX = (Integer) objectMap.get("PosX"); + int posZ = (Integer) objectMap.get("PosZ"); + DhSectionPos pos = new DhSectionPos(sectionDetailLevel, posX, posZ); + + int minY = (Integer) objectMap.get("MinY"); + int dataChecksum = (Integer) objectMap.get("DataChecksum"); + + byte[] dataByteArray = (byte[]) objectMap.get("Data"); + byte[] columnGenStepByteArray = (byte[]) objectMap.get("ColumnGenerationStep"); + byte[] mappingByteArray = (byte[]) objectMap.get("Mapping"); + + + byte dataFormatVersion = (Byte) objectMap.get("DataFormatVersion"); + + boolean applyToParent = ((int) objectMap.get("ApplyToParent")) == 1; + + long lastModifiedUnixDateTime = (Long) objectMap.get("LastModifiedUnixDateTime"); + long createdUnixDateTime = (Long) objectMap.get("CreatedUnixDateTime"); + + NewFullDataSourceDTO dto = new NewFullDataSourceDTO( + pos, + dataChecksum, columnGenStepByteArray, dataFormatVersion, dataByteArray, + lastModifiedUnixDateTime, createdUnixDateTime, + mappingByteArray, applyToParent, + minY); + return dto; + } + + @Override + public PreparedStatement createInsertStatement(NewFullDataSourceDTO dto) throws SQLException + { + String sql = + "INSERT INTO "+this.getTableName() + " (\n" + + " DetailLevel, PosX, PosZ, \n" + + " MinY, DataChecksum, \n" + + " Data, ColumnGenerationStep, Mapping, \n" + + " DataFormatVersion, ApplyToParent, \n" + + " LastModifiedUnixDateTime, CreatedUnixDateTime) \n" + + "VALUES( \n" + + " ?, ?, ?, \n" + + " ?, ?, \n" + + " ?, ?, ?, \n" + + " ?, ?, \n" + + " ?, ? \n" + + ");"; + PreparedStatement statement = this.createPreparedStatement(sql); + + int i = 1; + statement.setObject(i++, dto.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + statement.setObject(i++, dto.pos.getX()); + statement.setObject(i++, dto.pos.getZ()); + + statement.setObject(i++, dto.levelMinY); + statement.setObject(i++, dto.dataChecksum); + + statement.setObject(i++, dto.dataByteArray); + statement.setObject(i++, dto.columnGenStepByteArray); + statement.setObject(i++, dto.mappingByteArray); + + statement.setObject(i++, dto.dataFormatVersion); + statement.setObject(i++, dto.applyToParent); + + statement.setObject(i++, dto.lastModifiedUnixDateTime); + statement.setObject(i++, dto.createdUnixDateTime); + + return statement; + } + + @Override + public PreparedStatement createUpdateStatement(NewFullDataSourceDTO dto) throws SQLException + { + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET \n" + + " MinY = ? \n" + + " ,DataChecksum = ? \n" + + + " ,Data = ? \n" + + " ,ColumnGenerationStep = ? \n" + + " ,Mapping = ? \n" + + + " ,DataFormatVersion = ? \n" + + " ,ApplyToParent = ? \n" + + + " ,LastModifiedUnixDateTime = ? \n" + + " ,CreatedUnixDateTime = ? \n" + + + "WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?"; + PreparedStatement statement = this.createPreparedStatement(sql); + + int i = 1; + statement.setObject(i++, dto.levelMinY); + statement.setObject(i++, dto.dataChecksum); + + statement.setObject(i++, dto.dataByteArray); + statement.setObject(i++, dto.columnGenStepByteArray); + statement.setObject(i++, dto.mappingByteArray); + + statement.setObject(i++, dto.dataFormatVersion); + statement.setObject(i++, dto.applyToParent); + + statement.setObject(i++, dto.lastModifiedUnixDateTime); + statement.setObject(i++, dto.createdUnixDateTime); + + statement.setObject(i++, dto.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + statement.setObject(i++, dto.pos.getX()); + statement.setObject(i++, dto.pos.getZ()); + + return statement; + } + + + + // updates // + + public void setApplyToParent(DhSectionPos pos, boolean applyToParent) throws SQLException + { + int detailLevel = pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET ApplyToParent = "+applyToParent+" \n" + + "WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ(); + + this.queryDictionaryFirst(sql); + } + + public ArrayList getPositionsToUpdate(int returnCount) + { + ArrayList list = new ArrayList<>(); + + List> resultMapList = this.queryDictionary( + "select DetailLevel, PosX, PosZ " + + "from "+this.getTableName()+" " + + "where ApplyToParent = 1 " + + "order by DetailLevel asc LIMIT "+returnCount+";"); + + for (Map resultMap : resultMapList) + { + byte detailLevel = (Byte) resultMap.get("DetailLevel"); + byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + int posX = (Integer) resultMap.get("PosX"); + int posZ = (Integer) resultMap.get("PosZ"); + + DhSectionPos pos = new DhSectionPos(sectionDetailLevel, posX, posZ); + list.add(pos); + } + + return list; + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java similarity index 91% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java index 7e2aee31e..fa43e81ea 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java @@ -17,13 +17,13 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.sql.SQLException; -public class RenderDataRepo extends AbstractDataSourceRepo +public class RenderDataRepo extends AbstractLegacyDataSourceRepo { public static final String TABLE_NAME = "DhRenderData"; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index 606af3eb6..9360c3ac5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -118,6 +118,9 @@ public class FullDataPointUtil { // this try-catch is present to fix an issue where the stack trace is missing // and to allow for easily attaching a debugger + + // if this was thrown that probably means the datasource has been + // re-mapped multiple times, causing the ID's to go out of their expected bounds. throw new RuntimeException(e); } } diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql new file mode 100644 index 000000000..e78cff45c --- /dev/null +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql @@ -0,0 +1,25 @@ + +--batch-- + +CREATE TABLE FullData ( + -- compound primary key + DetailLevel TINYINT NOT NULL -- LOD detail level, not section detail level IE 0, 1, 2 not 6, 7, 8 + ,PosX INT NOT NULL + ,PosZ INT NOT NULL + + ,MinY INT NOT NULL + ,DataChecksum INT NOT NULL + + ,Data BLOB NULL + ,ColumnGenerationStep BLOB NULL + ,Mapping BLOB NULL + + ,DataFormatVersion TINYINT NULL + + ,ApplyToParent BIT NULL + + ,LastModifiedUnixDateTime BIGINT NOT NULL -- in GMT 0 + ,CreatedUnixDateTime BIGINT NOT NULL -- in GMT 0 + + ,PRIMARY KEY (DetailLevel, PosX, PosZ) +); diff --git a/core/src/main/resources/sqlScripts/scriptList.txt b/core/src/main/resources/sqlScripts/scriptList.txt index 47c749378..2a2f2e905 100644 --- a/core/src/main/resources/sqlScripts/scriptList.txt +++ b/core/src/main/resources/sqlScripts/scriptList.txt @@ -1,2 +1,3 @@ 0010-sqlite-createInitialDataTables.sql +0020-sqlite-createGeneratedFullDataSourceTables.sql diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyDto.java b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java index fbabbe52e..745014dec 100644 --- a/core/src/test/java/testItems/sql/TestCompoundKeyDto.java +++ b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java @@ -20,8 +20,7 @@ package testItems.sql; import com.seibel.distanthorizons.core.pos.DhChunkPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.IBaseDTO; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; public class TestCompoundKeyDto implements IBaseDTO { diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java index 114e4b57d..3b7ed530e 100644 --- a/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java +++ b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java @@ -20,7 +20,7 @@ package testItems.sql; import com.seibel.distanthorizons.core.pos.DhChunkPos; -import com.seibel.distanthorizons.core.sql.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java index 748c3a214..347e6d686 100644 --- a/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java +++ b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java @@ -19,7 +19,7 @@ package testItems.sql; -import com.seibel.distanthorizons.core.sql.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/core/src/test/java/testItems/sql/TestSingleKeyDto.java b/core/src/test/java/testItems/sql/TestSingleKeyDto.java index 6de1352f3..90709a162 100644 --- a/core/src/test/java/testItems/sql/TestSingleKeyDto.java +++ b/core/src/test/java/testItems/sql/TestSingleKeyDto.java @@ -19,7 +19,7 @@ package testItems.sql; -import com.seibel.distanthorizons.core.sql.IBaseDTO; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; public class TestSingleKeyDto implements IBaseDTO { diff --git a/core/src/test/java/tests/DhRepoSqliteTest.java b/core/src/test/java/tests/DhRepoSqliteTest.java index 8d15c5c6a..066106110 100644 --- a/core/src/test/java/tests/DhRepoSqliteTest.java +++ b/core/src/test/java/tests/DhRepoSqliteTest.java @@ -21,6 +21,7 @@ package tests; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.sql.DatabaseUpdater; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -34,7 +35,7 @@ import java.sql.SQLException; import java.util.Map; /** - * Validates {@link com.seibel.distanthorizons.core.sql.AbstractDhRepo} is set up correctly. + * Validates {@link AbstractDhRepo} is set up correctly. */ public class DhRepoSqliteTest {