From 23e857a20df78a816eeae88abc8db4058361e309 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 27 Apr 2024 11:35:08 -0500 Subject: [PATCH 01/16] Fix some lib shading issues --- core/build.gradle | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 455d9e23d..7070d752d 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -61,14 +61,16 @@ dependencies { // All of these dependencies are in Vanilla Minecraft, but we nee shade("com.electronwill.night-config:toml:${rootProject.nightconfig_version}") shade("com.electronwill.night-config:json:${rootProject.nightconfig_version}") - - // needed for the standalone jar - shade("org.apache.logging.log4j:log4j-core:2.23.1") - shade("org.apache.logging.log4j:log4j-api:2.23.1") // SVG (not needed atm) //shade("com.formdev:svgSalamander:${rootProject.svgSalamander_version}") + // can't be included since relocation is broken due to + // reflection Log4j does on their end. + // TODO create our own logging interface to work with/without log4j for use as a standalone jar or with MC +// shade("org.apache.logging.log4j:log4j-api:${rootProject.log4j_version}") +// shade("org.apache.logging.log4j:log4j-core:${rootProject.log4j_version}") + @@ -93,31 +95,16 @@ shadowJar { relocate "it.unimi.dsi.fastutil", "${librariesLocation}.unimi.dsi.fastutil" - // LWJGL - // Only ever shadow the dependencies we use otherwise some stuff would break when running on an external client - relocate "org.lwjgl.system.jawt", "${librariesLocation}.lwjgl.system.jawt" - - // Compression + // Compression (LZ4) relocate "net.jpountz", "${librariesLocation}.jpountz" relocate "com.github.luben", "${librariesLocation}.github.luben" relocate "org.tukaani", "${librariesLocation}.tukaani" - // Sqlite Database - //At the moment, there is a bug in this library which doesnt allow it to be relocated -// relocate "org.sqlite", "${librariesLocation}.sqlite" - - // JOML - if (project.hasProperty("embed_joml") && embed_joml == "true") - relocate "org.joml", "${librariesLocation}.joml" - - // NightConfig (includes Toml & Json) + // night config relocate "com.electronwill.nightconfig", "${librariesLocation}.electronwill.nightconfig" - + // Netty relocate "io.netty", "${librariesLocation}.netty" - - relocate "org.apache.logging", "${librariesLocation}.apache.logging" - mergeServiceFiles() } From 7f874b4dc53c05cdcc109096454ba81089c94391 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 27 Apr 2024 12:56:08 -0500 Subject: [PATCH 02/16] Revert a613540b --- core/build.gradle | 52 +------------------ .../distanthorizons/core/Initializer.java | 4 +- 2 files changed, 4 insertions(+), 52 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 7070d752d..8e8bf884f 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -39,39 +39,8 @@ dependencies { // All of these dependencies are in Vanilla Minecraft, but we nee runtimeOnly "org.lwjgl:lwjgl-opengl::$lwjglNatives" runtimeOnly "org.lwjgl:lwjgl-stb::$lwjglNatives" runtimeOnly "org.lwjgl:lwjgl-tinyfd::$lwjglNatives" - - - // fast util - shade("it.unimi.dsi:fastutil:${rootProject.fastutil_version}") - - // Compression - // needs to be here and in core to prevent runtime/compiler errors - shade("org.lz4:lz4-java:${rootProject.lz4_version}") // LZ4 - shade("com.github.luben:zstd-jni:${rootProject.zstd_version}") // Zstd - shade("org.tukaani:xz:${rootProject.xz_version}") // LZMA - - // Sqlite Database - shade("org.xerial:sqlite-jdbc:${rootProject.sqlite_jdbc_version}") - - // Netty - shade("io.netty:netty-all:${rootProject.netty_version}") - - // NightConfig (includes Toml & Json) - // needs to be here and in core to prevent runtime/compiler errors - shade("com.electronwill.night-config:toml:${rootProject.nightconfig_version}") - shade("com.electronwill.night-config:json:${rootProject.nightconfig_version}") - - - // SVG (not needed atm) - //shade("com.formdev:svgSalamander:${rootProject.svgSalamander_version}") - - // can't be included since relocation is broken due to - // reflection Log4j does on their end. - // TODO create our own logging interface to work with/without log4j for use as a standalone jar or with MC -// shade("org.apache.logging.log4j:log4j-api:${rootProject.log4j_version}") -// shade("org.apache.logging.log4j:log4j-core:${rootProject.log4j_version}") - + shade "it.unimi.dsi:fastutil:${rootProject.fastutil_version}" // Add our own fastutil version // Some other dependencies @@ -83,28 +52,11 @@ dependencies { // All of these dependencies are in Vanilla Minecraft, but we nee } artifacts { - shade shadowJar shadowedArtifact shadowJar // Setup the configuration shadowedArtifact to be the shadowJar } shadowJar { - configurations = [project.configurations.shade] - def librariesLocation = "distanthorizons.libraries" - - relocate "it.unimi.dsi.fastutil", "${librariesLocation}.unimi.dsi.fastutil" - - // Compression (LZ4) - relocate "net.jpountz", "${librariesLocation}.jpountz" - relocate "com.github.luben", "${librariesLocation}.github.luben" - relocate "org.tukaani", "${librariesLocation}.tukaani" - - // night config - relocate "com.electronwill.nightconfig", "${librariesLocation}.electronwill.nightconfig" - - // Netty - relocate "io.netty", "${librariesLocation}.netty" - mergeServiceFiles() -} +} \ No newline at end of file diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index b84c721ef..1b4367a3a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -28,7 +28,7 @@ import com.seibel.distanthorizons.core.api.external.methods.config.DhApiConfig; import com.seibel.distanthorizons.core.api.external.methods.data.DhApiTerrainDataRepo; import com.seibel.distanthorizons.api.DhApi; import com.seibel.distanthorizons.core.render.DhApiRenderProxy; -import io.netty.buffer.ByteBuf; +//import io.netty.buffer.ByteBuf; import net.jpountz.lz4.LZ4FrameOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -48,7 +48,7 @@ public class Initializer // if any library isn't present in the jar its class // will throw an error (not an exception) Class compressor = LZ4FrameOutputStream.class; - Class networking = ByteBuf.class; + //Class networking = ByteBuf.class; Class toml = com.electronwill.nightconfig.core.Config.class; } catch (Throwable e) From 3b600ce800de5eb2a26cad98bb8ba5c9cd62871b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 28 Apr 2024 15:52:08 -0500 Subject: [PATCH 03/16] Add corrupt data read handling --- .../fullData/FullDataPointIdMap.java | 25 ++- .../fullData/sources/FullDataSourceV1.java | 7 +- .../fullData/sources/FullDataSourceV2.java | 22 ++- .../transformers/LodDataBuilder.java | 173 +++++++++--------- .../core/file/AbstractDataSourceHandler.java | 18 +- .../FullDataSourceProviderV1.java | 10 +- .../FullDataSourceProviderV2.java | 5 +- .../core/generation/WorldGenerationQueue.java | 6 + .../core/sql/dto/FullDataSourceV2DTO.java | 47 +++-- .../core/util/FullDataPointUtil.java | 55 ++++-- .../util/objects/DataCorruptedException.java | 24 +++ 11 files changed, 262 insertions(+), 130 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index ce362a4e6..61a29f3e5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -259,9 +260,14 @@ public class FullDataPointIdMap } /** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */ - public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { int entityCount = inputStream.readInt(); + if (entityCount < 0) + { + throw new DataCorruptedException("FullDataPointIdMap deserialize entry count should have a number greater than or equal to 0, returned value ["+entityCount+"]."); + } + // only used when debugging HashMap dataPointEntryBySerialization = new HashMap<>(); @@ -269,6 +275,13 @@ public class FullDataPointIdMap FullDataPointIdMap newMap = new FullDataPointIdMap(pos); for (int i = 0; i < entityCount; i++) { + // necessary to prevent issues with deserializing objects after the level has been closed + if (Thread.interrupted()) + { + throw new InterruptedException(FullDataPointIdMap.class.getSimpleName() + " task interrupted."); + } + + String entryString = inputStream.readUTF(); Entry newEntry = Entry.deserialize(entryString, levelWrapper); newMap.entryList.add(newEntry); @@ -457,18 +470,12 @@ public class FullDataPointIdMap public String serialize() { return this.biome.getSerialString() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString(); } - public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, DataCorruptedException { String[] stringArray = str.split(BLOCK_STATE_SEPARATOR_STRING); if (stringArray.length != 2) { - throw new IOException("Failed to deserialize BiomeBlockStateEntry"); - } - - // necessary to prevent issues with deserializing objects after the level has been closed - if (Thread.interrupted()) - { - throw new InterruptedException(FullDataPointIdMap.class.getSimpleName() + " task interrupted."); + throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry"); } IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapper(stringArray[0], levelWrapper); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 37cdf0452..4f782837b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; @@ -152,7 +153,7 @@ public class FullDataSourceV1 implements IDataSource * Clears and then overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an existing {@link FullDataSourceV1} and can be used in place of a constructor to reuse an existing {@link FullDataSourceV1} object. */ - public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException, DataCorruptedException { // clear/overwrite the old data this.resizeDataStructuresForRepopulation(dto.pos); @@ -166,7 +167,7 @@ public class FullDataSourceV1 implements IDataSource * Overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an empty {@link FullDataSourceV1} and functions similar to a constructor. */ - public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException, DataCorruptedException { FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level); this.setSourceSummaryData(summaryData); @@ -361,7 +362,7 @@ public class FullDataSourceV1 implements IDataSource outputStream.writeInt(DATA_GUARD_BYTE); this.mapping.serialize(outputStream); } - public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { int guardByte = inputStream.readInt(); if (guardByte != DATA_GUARD_BYTE) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 8846ba085..6db6b6411 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -32,6 +32,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; import it.unimi.dsi.fastutil.longs.LongArrayList; @@ -616,7 +617,16 @@ public class FullDataSourceV2 implements IDataSource { if (height != 0) { - newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + try + { + long datapoint = FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight); + newColumnList.add(datapoint); + } + catch (DataCorruptedException e) + { + // shouldn't happen, (especially if validation is disabled) but just in case + LOGGER.warn("Skipping corrupt datapoint for pos "+inputDataSource.pos+" at relative position ["+x+","+z+"] with data: ID["+lastId+"], Height["+height+"], minY["+minY+"], lastBlockLight["+lastBlockLight+"], lastSkyLight["+lastSkyLight+"]."); + } } lastId = id; @@ -630,7 +640,15 @@ public class FullDataSourceV2 implements IDataSource // add the last slice if present if (height != 0) { - newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + try + { + newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + } + catch (DataCorruptedException e) + { + // shouldn't happen, (especially if validation is disabled) but just in case + LOGGER.warn("Skipping corrupt datapoint for pos "+inputDataSource.pos+" at relative position ["+x+","+z+"] with data: ID["+lastId+"], Height["+height+"], minY["+minY+"], lastBlockLight["+lastBlockLight+"], lastSkyLight["+lastSkyLight+"]."); + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 9437ec217..bc6da58f6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; @@ -133,98 +134,106 @@ public class LodDataBuilder EDhApiWorldCompressionMode worldCompressionMode = Config.Client.Advanced.LodBuilding.worldCompression.get(); boolean ignoreHiddenBlocks = (worldCompressionMode != EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS); - int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); - for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) + try { - for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++) + int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); + for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) { - LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); - int lastY = chunkWrapper.getMaxBuildHeight(); - IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ); - IBlockStateWrapper blockState = AIR; - int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - - - byte blockLight; - byte skyLight; - if (lastY < chunkWrapper.getMaxBuildHeight()) + for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++) { - // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting - blockLight = (byte) chunkWrapper.getBlockLight(relBlockX, lastY + 1, relBlockZ); - skyLight = (byte) chunkWrapper.getSkyLight(relBlockX, lastY + 1, relBlockZ); - } - else - { - //we are at the height limit. There are no torches here, and sky is not obscured. - blockLight = 0; - skyLight = 15; - } - - - // determine the starting Y Pos - int y = chunkWrapper.getLightBlockingHeightMapValue(relBlockX,relBlockZ); - // go up until we reach open air or the world limit - IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); - while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight()) - { - try - { - // This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light. - // Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks. - y++; - topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); - } - catch (Exception e) - { - if (!getTopErrorLogged) - { - LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + relBlockX + "," + y + "," + relBlockZ + "] error: " + e.getMessage(), e); - getTopErrorLogged = true; - } - - y--; - break; - } - } - - - for (; y >= minBuildHeight; y--) - { - IBiomeWrapper newBiome = chunkWrapper.getBiome(relBlockX, y, relBlockZ); - IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); - byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ); - byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ); + LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); + int lastY = chunkWrapper.getMaxBuildHeight(); + IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ); + IBlockStateWrapper blockState = AIR; + int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - // save the biome/block change - if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) + + byte blockLight; + byte skyLight; + if (lastY < chunkWrapper.getMaxBuildHeight()) { - // if we ignore hidden blocks, don't save this biome/block change - // wait until the block is visible and then save the new datapoint - if (!ignoreHiddenBlocks - // if the last block is air, this block will always be visible - || blockState.isAir() - // check if this block is visible from any direction - || blockVisible(chunkWrapper, relBlockX, y, relBlockZ)) + // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting + blockLight = (byte) chunkWrapper.getBlockLight(relBlockX, lastY + 1, relBlockZ); + skyLight = (byte) chunkWrapper.getSkyLight(relBlockX, lastY + 1, relBlockZ); + } + else + { + //we are at the height limit. There are no torches here, and sky is not obscured. + blockLight = 0; + skyLight = 15; + } + + + // determine the starting Y Pos + int y = chunkWrapper.getLightBlockingHeightMapValue(relBlockX, relBlockZ); + // go up until we reach open air or the world limit + IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); + while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight()) + { + try { - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - biome = newBiome; - blockState = newBlockState; - mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - blockLight = newBlockLight; - skyLight = newSkyLight; - lastY = y; + // This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light. + // Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks. + y++; + topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); + } + catch (Exception e) + { + if (!getTopErrorLogged) + { + LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + relBlockX + "," + y + "," + relBlockZ + "] error: " + e.getMessage(), e); + getTopErrorLogged = true; + } + + y--; + break; } } + + + for (; y >= minBuildHeight; y--) + { + IBiomeWrapper newBiome = chunkWrapper.getBiome(relBlockX, y, relBlockZ); + IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); + byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ); + byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ); + + // save the biome/block change + if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) + { + // if we ignore hidden blocks, don't save this biome/block change + // wait until the block is visible and then save the new datapoint + if (!ignoreHiddenBlocks + // if the last block is air, this block will always be visible + || blockState.isAir() + // check if this block is visible from any direction + || blockVisible(chunkWrapper, relBlockX, y, relBlockZ)) + { + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + biome = newBiome; + blockState = newBlockState; + mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); + blockLight = newBlockLight; + skyLight = newSkyLight; + lastY = y; + } + } + } + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + + dataSource.setSingleColumn(longs, + relBlockX + chunkOffsetX, + relBlockZ + chunkOffsetZ, + EDhApiWorldGenerationStep.LIGHT, + worldCompressionMode); } - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - - dataSource.setSingleColumn(longs, - relBlockX + chunkOffsetX, - relBlockZ + chunkOffsetZ, - EDhApiWorldGenerationStep.LIGHT, - worldCompressionMode); } } + catch (DataCorruptedException e) + { + LOGGER.error("Unable to convert chunk at pos ["+chunkWrapper.getChunkPos()+"] to an LOD. Error: "+e.getMessage(), e); + return null; + } LodUtil.assertTrue(!dataSource.isEmpty); return dataSource; @@ -292,7 +301,7 @@ public class LodDataBuilder /** @throws ClassCastException if an API user returns the wrong object type(s) */ - public static FullDataSourceV2 createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException + public static FullDataSourceV2 createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException, DataCorruptedException { FullDataSourceV2 accessor = FullDataSourceV2.createEmpty(new DhSectionPos(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ))); for (int relZ = 0; relZ < LodUtil.CHUNK_WIDTH; relZ++) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index b913e16c7..c89168b75 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -8,6 +8,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; @@ -16,6 +17,7 @@ import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; +import java.io.StreamCorruptedException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; @@ -95,7 +97,7 @@ public abstract class AbstractDataSourceHandler /** When this is called the parent folders should be created */ protected abstract TRepo createRepo(); - protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException; + protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException, DataCorruptedException; protected abstract TDTO createDtoFromDataSource(TDataSource dataSource); protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); @@ -145,8 +147,18 @@ public abstract class AbstractDataSourceHandler TDTO dto = this.repo.getByKey(pos); if (dto != null) { - // load from database - dataSource = this.createDataSourceFromDto(dto); + try + { + // load from database + dataSource = this.createDataSourceFromDto(dto); + } + catch (DataCorruptedException e) + { + // stack trace not included since a lot of corrupt data would cause the log to get quite messy, + // and it should be fairly easy to see what the problem was from the message + LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage()); + this.repo.deleteWithKey(pos); + } } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index fde42a6a7..1ee291fcc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -7,6 +7,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV1Repo; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -72,7 +73,7 @@ public class FullDataSourceProviderV1 } } - protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException + protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException, DataCorruptedException { FullDataSourceV1 dataSource = FullDataSourceV1.createEmpty(dto.pos); dataSource.populateFromStream(dto, dto.getInputStream(), this.level); @@ -128,6 +129,13 @@ public class FullDataSourceProviderV1 } } catch (InterruptedException ignore) { } + catch (DataCorruptedException e) + { + // stack trace not included since a lot of corrupt data would cause the log to get quite messy, + // and it should be fairly easy to see what the problem was from the message + LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage()); + this.repo.deleteWithKey(pos); + } catch (IOException e) { LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 795ffb15c..5bafbc87e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.Logger; @@ -165,7 +166,7 @@ public class FullDataSourceProviderV2 } @Override - protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException + protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException { return dto.createPooledDataSource(this.level.getLevelWrapper()); } @Override @@ -263,7 +264,7 @@ public class FullDataSourceProviderV2 } catch (Exception e) { - LOGGER.error("issue in update for parent pos: " + parentUpdatePos); + LOGGER.error("issue in update for parent pos: " + parentUpdatePos+ " Error: "+e.getMessage(), e); } finally { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 04c485bf2..da1595893 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -36,6 +36,7 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException; import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; @@ -469,6 +470,11 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb FullDataSourceV2 dataSource = LodDataBuilder.createFromApiChunkData(dataPoints); chunkDataConsumer.accept(dataSource); } + catch (DataCorruptedException e) + { + LOGGER.error("World generator returned a corrupt chunk. Error: [" + e.getMessage() + "]. World generator disabled.", e); + Config.Client.Advanced.WorldGenerator.enableDistantGeneration.set(false); + } catch (ClassCastException e) { LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index e59007129..bdab7dd0d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -25,21 +25,24 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.jetbrains.annotations.NotNull; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; +import java.io.*; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; /** handles storing {@link FullDataSourceV2}'s in the database. */ public class FullDataSourceV2DTO implements IBaseDTO { + public static final boolean VALIDATE_INPUT_DATAPOINTS = true; + + public DhSectionPos pos; public int levelMinY; @@ -118,23 +121,23 @@ public class FullDataSourceV2DTO implements IBaseDTO // data source population // //========================// - public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(this.pos, false); return this.populateDataSource(dataSource, levelWrapper); } - public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } /** * May be missing one or more data fields.
* Designed to be used without access to Minecraft or any supporting objects. */ - public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException + public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, true); } - private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException + private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException { if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) { @@ -212,7 +215,7 @@ public class FullDataSourceV2DTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); @@ -225,12 +228,21 @@ public class FullDataSourceV2DTO implements IBaseDTO { // read the column length short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later + if (dataColumnLength < 0) + { + throw new DataCorruptedException("Read DataSource Blob data at index ["+xz+"], column length ["+dataColumnLength+"] should be greater than zero."); + } + LongArrayList dataColumn = new LongArrayList(new long[dataColumnLength]); // read column data (will be skipped if no data was present) for (int y = 0; y < dataColumnLength; y++) { long dataPoint = compressedIn.readLong(); + if (VALIDATE_INPUT_DATAPOINTS) + { + FullDataPointUtil.validateDatapoint(dataPoint); + } dataColumn.set(y, dataPoint); } @@ -254,15 +266,22 @@ public class FullDataSourceV2DTO implements IBaseDTO return byteArrayOutputStream.toByteArray(); } - private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); - byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; - compressedIn.readFully(columnGenStepByteArray); - - return columnGenStepByteArray; + try + { + byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + compressedIn.readFully(columnGenStepByteArray); + + return columnGenStepByteArray; + } + catch (EOFException e) + { + throw new DataCorruptedException(e); + } } @@ -302,7 +321,7 @@ public class FullDataSourceV2DTO implements IBaseDTO return byteArrayOutputStream.toByteArray(); } - private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index b63c6f0e7..2b30f66c2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -2,6 +2,7 @@ package com.seibel.distanthorizons.core.util; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.coreapi.ModInfo; import org.jetbrains.annotations.Contract; @@ -72,23 +73,11 @@ public class FullDataPointUtil * creates a new datapoint with the given values * @param relMinY relative to the minimum level Y value */ - public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) + public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) throws DataCorruptedException { if (RUN_VALIDATION) { - // assertions are inside if-blocks to prevent unnecessary string concatenations - if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with y[" + relMinY + "] out of range!"); - } - if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with height[" + height + "] out of range!"); - } - if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with y+depth[" + (relMinY + height) + "] out of range!"); - } + validateData(id, height, relMinY, blockLight, skyLight); } @@ -116,6 +105,44 @@ public class FullDataPointUtil return data; } + public static void validateDatapoint(long datapoint) throws DataCorruptedException { validateData(getId(datapoint), getHeight(datapoint), getBottomY(datapoint), (byte)getBlockLight(datapoint), (byte)getSkyLight(datapoint)); } + /** + * Throws {@link DataCorruptedException} if any of the given values are outside + * their expected range. + */ + public static void validateData(int id, int height, int relMinY, byte blockLight, byte skyLight) throws DataCorruptedException + { + // ID + if (id < 0) + { + throw new DataCorruptedException("Full datapoint ID [" + relMinY + "] must be greater than zero."); + } + + // height + if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + throw new DataCorruptedException("Full datapoint relative min y [" + relMinY + "] must be in the range [0 - "+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"] (inclusive)."); + } + if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + throw new DataCorruptedException("Full datapoint height [" + height + "] must be in the range [1 - "+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"] (inclusive)."); + } + if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + throw new DataCorruptedException("Full datapoint y+depth [" + (relMinY + height) + "] is higher than the maximum world Y height ["+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"]."); + } + + // lighting + if (blockLight < LodUtil.MIN_MC_LIGHT || blockLight > LodUtil.MAX_MC_LIGHT) + { + throw new DataCorruptedException("Full datapoint block light [" + blockLight + "] must be in the range ["+LodUtil.MIN_MC_LIGHT+" - "+LodUtil.MAX_MC_LIGHT+"] (inclusive)."); + } + if (skyLight < LodUtil.MIN_MC_LIGHT || skyLight > LodUtil.MAX_MC_LIGHT) + { + throw new DataCorruptedException("Full datapoint sky light [" + skyLight + "] must be in the range ["+LodUtil.MIN_MC_LIGHT+" - "+LodUtil.MAX_MC_LIGHT+"] (inclusive)."); + } + } + /** Returns the BlockState/Biome pair ID used to identify this LOD's color */ public static int getId(long data) { return (int) (data & ID_MASK); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java new file mode 100644 index 000000000..3195bb320 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java @@ -0,0 +1,24 @@ +package com.seibel.distanthorizons.core.util.objects; + +/** + * Thrown when a DH handled resource or datasource isn't in the + * correct format.

+ * + * IE: a blocklight with the value -4 when it should be between 0 and 15. + */ +public class DataCorruptedException extends Exception +{ + /** replaces this exception's stack trace with the incoming one */ + public DataCorruptedException(Exception e) + { + super(e.getMessage()); + this.setStackTrace(e.getStackTrace()); + this.addSuppressed(e); + } + + public DataCorruptedException(String message) + { + super(message); + } + +} From c83140a2d002022f938e5521a495764c0e435b56 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 28 Apr 2024 17:31:08 -0500 Subject: [PATCH 04/16] add IClientLevelWrapper.getPlainsBiomeWrapper() --- .../core/wrapperInterfaces/world/IClientLevelWrapper.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java index 60d74d4bf..9aef5d00c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java @@ -37,4 +37,8 @@ public interface IClientLevelWrapper extends ILevelWrapper /** @return -1 if there was a problem getting the color */ int getDirtBlockColor(); + /** Will return null if there was an issue finding the biome. */ + @Nullable + IBiomeWrapper getPlainsBiomeWrapper(); + } From 8ecd5dd9cbf57f0e8ba7cbaa75289d5270176027 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 30 Apr 2024 18:57:07 -0500 Subject: [PATCH 05/16] Fix optifine 1.16 support --- .../distanthorizons/core/render/renderer/LodRenderer.java | 7 ++++--- .../core/wrapperInterfaces/misc/ILightMapWrapper.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index 584e275f1..bc754b520 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -57,6 +57,9 @@ import org.apache.logging.log4j.LogManager; import org.lwjgl.opengl.GL32; import java.awt.*; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.time.Duration; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @@ -425,7 +428,7 @@ public class LodRenderer combinedMatrix.multiply(renderEventParam.dhModelViewMatrix); // Note: this can be very slow if a lot of boxes are being rendered - DebugRenderer.INSTANCE.render(combinedMatrix); + DebugRenderer.INSTANCE.render(combinedMatrix); profiler.popPush("LOD cleanup"); } @@ -591,8 +594,6 @@ public class LodRenderer { if (this.usingMcFrameBuffer && framebufferOverride == null) { - // recreating the GL State at this point is necessary in order to get the correct depth texture - minecraftGlState.saveState(); if (ENABLE_DUMP_GL_STATE) { tickLogger.debug("Re-saving GL state due to Optifine presence: " + minecraftGlState); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/misc/ILightMapWrapper.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/misc/ILightMapWrapper.java index d16cbfa44..9d601653b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/misc/ILightMapWrapper.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/misc/ILightMapWrapper.java @@ -28,7 +28,7 @@ import com.seibel.distanthorizons.coreapi.interfaces.dependencyInjection.IBindab public interface ILightMapWrapper extends IBindable { - // Returns the binded texture position + /** Returns the bound texture position */ void bind(); void unbind(); From f0506d28e50cf8600bc886e815467dc5a9abbd4e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 30 Apr 2024 20:26:34 -0500 Subject: [PATCH 06/16] Fix incorrect refernce to fastutil in LzmaArrayCache --- .../core/util/objects/dataStreams/LzmaArrayCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java index e932e9936..5c982130c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java @@ -2,12 +2,12 @@ package com.seibel.distanthorizons.core.util.objects.dataStreams; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import it.unimi.dsi.fastutil.ints.Int2ReferenceArrayMap; -import it.unimi.dsi.fastutil.ints.IntUnaryOperator; import org.apache.logging.log4j.Logger; import org.tukaani.xz.ArrayCache; import java.util.ArrayList; import java.util.Arrays; +import java.util.function.IntUnaryOperator; import java.util.concurrent.atomic.AtomicInteger; /** From 9196480e50b1c03d01c638de53b03119d99b0919 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 30 Apr 2024 20:28:34 -0500 Subject: [PATCH 07/16] Remove references to FastUtil 8.5.13 --- core/build.gradle | 5 +- .../distanthorizons/core/Initializer.java | 2 + .../core/util/AtomicsUtil.java | 141 ------------------ 3 files changed, 5 insertions(+), 143 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/AtomicsUtil.java diff --git a/core/build.gradle b/core/build.gradle index 8e8bf884f..7e3e9ac7a 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -40,7 +40,8 @@ dependencies { // All of these dependencies are in Vanilla Minecraft, but we nee runtimeOnly "org.lwjgl:lwjgl-stb::$lwjglNatives" runtimeOnly "org.lwjgl:lwjgl-tinyfd::$lwjglNatives" - shade "it.unimi.dsi:fastutil:${rootProject.fastutil_version}" // Add our own fastutil version + // FIXME for some reason this line doesn't actually shade in the library +// shade "it.unimi.dsi:fastutil:${rootProject.fastutil_version}" // Add our own fastutil version // Some other dependencies @@ -57,6 +58,6 @@ artifacts { shadowJar { def librariesLocation = "distanthorizons.libraries" - relocate "it.unimi.dsi.fastutil", "${librariesLocation}.unimi.dsi.fastutil" +// relocate "it.unimi.dsi.fastutil", "${librariesLocation}.unimi.dsi.fastutil" mergeServiceFiles() } \ No newline at end of file diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index 1b4367a3a..ede8a114b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -50,6 +50,8 @@ public class Initializer Class compressor = LZ4FrameOutputStream.class; //Class networking = ByteBuf.class; Class toml = com.electronwill.nightconfig.core.Config.class; + Class oldFastUtil = it.unimi.dsi.fastutil.longs.LongArrayList.class; // available in 8.2.1 + //Class newFastUtil = it.unimi.dsi.fastutil.ints.IntUnaryOperator.class; // available in 8.5.13 } catch (Throwable e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/AtomicsUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/AtomicsUtil.java deleted file mode 100644 index 99c3a6b59..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/AtomicsUtil.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.util; - -import it.unimi.dsi.fastutil.booleans.BooleanObjectImmutablePair; - -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.AtomicReferenceArray; -import java.util.function.Predicate; - -/** - * While java 8 does have built in atomic operations, there doesn't seem to be any Compare And Exchange operation...
- * So here we implement our own. - */ -public class AtomicsUtil -{ - - public static T conditionalAndExchange(AtomicReference atomic, Predicate requirement, T newValue) - { - while (true) - { - T oldValue = atomic.get(); - if (!requirement.test(oldValue)) return oldValue; - if (atomic.weakCompareAndSet(oldValue, newValue)) return oldValue; - } - } - - public static BooleanObjectImmutablePair conditionalAndExchangeWeak(AtomicReference atomic, Predicate requirement, T newValue) - { - T oldValue = atomic.get(); - if (requirement.test(oldValue) && atomic.weakCompareAndSet(oldValue, newValue)) - { - return new BooleanObjectImmutablePair<>(true, oldValue); - } - else - { - return new BooleanObjectImmutablePair<>(false, oldValue); - } - } - - /** - * If the {@link AtomicReference}'s current value matches the expected value, the newValue will be swapped in and the expected value returned.
- * If the {@link AtomicReference}'s current value DOESN'T match the expected value, the {@link AtomicReference}'s current value will be returned without modification. - */ - public static T compareAndExchange(AtomicReference atomic, T expected, T newValue) - { - while (true) - { - T oldValue = atomic.get(); - if (oldValue != expected) - { - return oldValue; - } - else if (atomic.weakCompareAndSet(expected, newValue)) - { - return expected; - } - } - } - - public static BooleanObjectImmutablePair compareAndExchangeWeak(AtomicReference atomic, T expected, T newValue) - { - T oldValue = atomic.get(); - if (oldValue == expected && atomic.weakCompareAndSet(expected, newValue)) - { - return new BooleanObjectImmutablePair<>(true, expected); - } - else - { - return new BooleanObjectImmutablePair<>(false, oldValue); - } - } - - // Additionally, we implement some helper methods for frequently used atomic operations. // - - // Compare with expected value and set new value if equal. Then return whatever value the atomic now contains. - public static T compareAndSetThenGet(AtomicReference atomic, T expected, T newValue) - { - while (true) - { - T oldValue = atomic.get(); - if (oldValue != expected) return oldValue; - if (atomic.weakCompareAndSet(expected, newValue)) return newValue; - } - } - - - - // Below is the array version of the above. // - - public static T compareAndExchange(AtomicReferenceArray array, int index, T expected, T newValue) - { - while (true) - { - T oldValue = array.get(index); - if (oldValue != expected) return oldValue; - if (array.weakCompareAndSet(index, expected, newValue)) return expected; - } - } - - public static BooleanObjectImmutablePair compareAndExchangeWeak(AtomicReferenceArray array, int index, T expected, T newValue) - { - T oldValue = array.get(index); - if (oldValue == expected && array.weakCompareAndSet(index, expected, newValue)) - { - return new BooleanObjectImmutablePair<>(true, expected); - } - else - { - return new BooleanObjectImmutablePair<>(false, oldValue); - } - } - - public static T compareAndSetThenGet(AtomicReferenceArray array, int index, T expected, T newValue) - { - while (true) - { - T oldValue = array.get(index); - if (oldValue != expected) return oldValue; - if (array.weakCompareAndSet(index, expected, newValue)) return newValue; - } - } - -} From c8f11548310017e852b9fd1bef3319b7d8d05abd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 30 Apr 2024 21:17:26 -0500 Subject: [PATCH 08/16] Remove ZStd compression option Any ZStd data will be automatically deleted and re-generated --- .../config/EDhApiDataCompressionMode.java | 8 ++-- .../distanthorizons/core/Initializer.java | 8 ++-- .../distanthorizons/core/config/Config.java | 6 --- .../core/file/AbstractDataSourceHandler.java | 13 +++++-- .../core/sql/dto/FullDataSourceV2DTO.java | 37 +++++++++++++++---- .../core/sql/repo/FullDataSourceV2Repo.java | 10 ++--- .../dataStreams/DhDataInputStream.java | 5 +-- .../dataStreams/DhDataOutputStream.java | 6 --- .../assets/distanthorizons/lang/en_us.json | 8 ++-- core/src/test/java/tests/CompressionTest.java | 12 +++--- 10 files changed, 64 insertions(+), 49 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java index ce277088c..4e093a234 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java @@ -56,14 +56,15 @@ public enum EDhApiDataCompressionMode */ LZ4(1), - /** + /* * Decent speed and good compression.

* * Read Speed: 11.78 MS / DTO
* Write Speed: 16.76 MS / DTO
* Compression ratio: 0.2199
*/ - Z_STD(2), + //@Deprecated + //Z_STD(2), /** * Extremely slow, but very good compression.

@@ -82,7 +83,8 @@ public enum EDhApiDataCompressionMode EDhApiDataCompressionMode(int value) { this.value = (byte) value; } - public static EDhApiDataCompressionMode getFromValue(byte value) + /** @throws IllegalArgumentException if the value doesn't map to a value */ + public static EDhApiDataCompressionMode getFromValue(byte value) throws IllegalArgumentException { EDhApiDataCompressionMode[] enumList = EDhApiDataCompressionMode.values(); for (int i = 0; i < enumList.length; i++) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index ede8a114b..76c2f0515 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -28,10 +28,10 @@ import com.seibel.distanthorizons.core.api.external.methods.config.DhApiConfig; import com.seibel.distanthorizons.core.api.external.methods.data.DhApiTerrainDataRepo; import com.seibel.distanthorizons.api.DhApi; import com.seibel.distanthorizons.core.render.DhApiRenderProxy; -//import io.netty.buffer.ByteBuf; import net.jpountz.lz4.LZ4FrameOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.tukaani.xz.XZOutputStream; import java.awt.*; @@ -47,15 +47,17 @@ public class Initializer { // if any library isn't present in the jar its class // will throw an error (not an exception) - Class compressor = LZ4FrameOutputStream.class; + Class fastCompressor = LZ4FrameOutputStream.class; + Class smallCompressor = XZOutputStream.class; //Class networking = ByteBuf.class; - Class toml = com.electronwill.nightconfig.core.Config.class; + Class config = com.electronwill.nightconfig.core.Config.class; Class oldFastUtil = it.unimi.dsi.fastutil.longs.LongArrayList.class; // available in 8.2.1 //Class newFastUtil = it.unimi.dsi.fastutil.ints.IntUnaryOperator.class; // available in 8.5.13 } catch (Throwable e) { LOGGER.fatal("Critical programmer error: One or more libraries aren't present. Error: [" + e.getMessage() + "]."); + // throwing here should crash the game, notifying the developer that something is wrong throw e; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index ac2f0bb33..0aaa4b94a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -771,12 +771,6 @@ public class Config + "Estimated average DTO read speed: 1.85 ms\n" + "Estimated average DTO write speed: 9.46 ms\n" + "\n" - + EDhApiDataCompressionMode.Z_STD + " \n" - + "A good middle ground between speed and compression.\n" - + "Expected Compression Ratio: 0.21\n" - + "Estimated average DTO read speed: 11.78 ms\n" - + "Estimated average DTO write speed: 16.77 ms\n" - + "\n" + EDhApiDataCompressionMode.LZMA2 + " \n" + "Slow but very good compression.\n" + "Expected Compression Ratio: 0.14\n" diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index c89168b75..207068e2b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -33,6 +33,8 @@ public abstract class AbstractDataSourceHandler implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final Set CORRUPT_DATA_ERRORS_LOGGED = Collections.newSetFromMap(new ConcurrentHashMap<>()); + /** * The highest numerical detail level possible. @@ -154,9 +156,14 @@ public abstract class AbstractDataSourceHandler } catch (DataCorruptedException e) { - // stack trace not included since a lot of corrupt data would cause the log to get quite messy, - // and it should be fairly easy to see what the problem was from the message - LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage()); + // Only log each message type once. + // This is done to prevent logging "No compression mode with the value [2]" 10,000 times + // if the user is migrating from a nightly build and used ZStd. + if (CORRUPT_DATA_ERRORS_LOGGED.add(e.getMessage())) + { + LOGGER.warn("Corrupted data found at pos " + pos + ". Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: " + e.getMessage(), e); + } + this.repo.deleteWithKey(pos); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index bdab7dd0d..728a6d350 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -60,7 +60,7 @@ public class FullDataSourceV2DTO implements IBaseDTO public byte[] compressedMappingByteArray; public byte dataFormatVersion; - public EDhApiDataCompressionMode compressionModeEnum; + public byte compressionModeValue; public boolean applyToParent; @@ -82,7 +82,7 @@ public class FullDataSourceV2DTO implements IBaseDTO return new FullDataSourceV2DTO( dataSource.getPos(), - checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, + checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum.value, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, dataSource.levelMinY @@ -91,7 +91,7 @@ public class FullDataSourceV2DTO implements IBaseDTO public FullDataSourceV2DTO( DhSectionPos pos, - int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray, + int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, byte compressionModeValue, byte[] compressedDataByteArray, long lastModifiedUnixDateTime, long createdUnixDateTime, byte[] compressedMappingByteArray, boolean applyToParent, int levelMinY) @@ -102,7 +102,7 @@ public class FullDataSourceV2DTO implements IBaseDTO this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray; this.dataFormatVersion = dataFormatVersion; - this.compressionModeEnum = compressionModeEnum; + this.compressionModeValue = compressionModeValue; this.compressedDataByteArray = compressedDataByteArray; this.compressedMappingByteArray = compressedMappingByteArray; @@ -144,9 +144,23 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"]."); } - dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); - dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, this.compressionModeEnum); - dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); + + EDhApiDataCompressionMode compressionModeEnum; + try + { + compressionModeEnum = this.getCompressionMode(); + } + catch (IllegalArgumentException e) + { + // may happen if ZStd was used (which was added and removed during the nightly builds) + // or if the compressor value is changed to an invalid option + throw new DataCorruptedException(e); + } + + + dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, compressionModeEnum); + dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, compressionModeEnum); + dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, compressionModeEnum); dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test @@ -157,7 +171,7 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests."); } - dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, this.compressionModeEnum)); + dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, compressionModeEnum)); } dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; @@ -341,6 +355,13 @@ public class FullDataSourceV2DTO implements IBaseDTO + //================// + // helper methods // + //================// + + public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } + + //================// // helper classes // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 619581a14..7f3df8519 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -24,6 +24,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; @@ -91,8 +92,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo Date: Tue, 30 Apr 2024 21:23:54 -0500 Subject: [PATCH 09/16] Fix debug wireframes rendering on top of LODs --- .../core/render/renderer/LodRenderer.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index bc754b520..157caf28c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -405,17 +405,10 @@ public class LodRenderer this.renderTransparentBuffers(profiler, renderEventParam, renderEventParam.partialTicks); } - - if (this.usingMcFrameBuffer) - { - // If MC's framebuffer is being used the depth needs to be cleared to prevent rendering on top of MC. - // This should only happen when Optifine shaders are being used. - GL32.glClear(GL32.GL_DEPTH_BUFFER_BIT); - } - drawLagSpikeCatcher.end("LodDraw"); + //=================// // debug rendering // //=================// @@ -434,6 +427,15 @@ public class LodRenderer + if (this.usingMcFrameBuffer) + { + // If MC's framebuffer is being used the depth needs to be cleared to prevent rendering on top of MC. + // This should only happen when Optifine shaders are being used. + GL32.glClear(GL32.GL_DEPTH_BUFFER_BIT); + } + + + //=============================// // Apply to the MC FrameBuffer // //=============================// From 6bb38ad500a8f94f5d8c9599b9c09c62575baa55 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 30 Apr 2024 21:52:13 -0500 Subject: [PATCH 10/16] Improve migration queue messages --- .../core/api/internal/ClientApi.java | 31 +++++++------ .../FullDataSourceProviderV2.java | 43 ++++++++++++++++++- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index c29fcfeef..ef9c06649 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -55,6 +55,8 @@ import org.lwjgl.glfw.GLFW; import java.util.HashMap; import java.util.HashSet; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; /** @@ -79,8 +81,7 @@ public class ClientApi private boolean configOverrideReminderPrinted = false; - private boolean migrationMessageShown = false; - private boolean showMigrationMessageNextFrame = false; + private final Queue chatMessageQueueForNextFrame = new LinkedBlockingQueue<>(); public boolean rendererDisabledBecauseOfExceptions = false; @@ -486,17 +487,16 @@ public class ClientApi MC.sendChatMessage(""); } - // data migration - if (this.showMigrationMessageNextFrame - && !this.migrationMessageShown - && Config.Client.Advanced.LodBuilding.showMigrationChatWarning.get()) + // generic messages + while (!this.chatMessageQueueForNextFrame.isEmpty()) { - this.showMigrationMessageNextFrame = false; - this.migrationMessageShown = true; - - MC.sendChatMessage("Old Distant Horizons data is being migrated."); - MC.sendChatMessage("During migration LODs may load slowly and DH world gen is disabled."); - MC.sendChatMessage(""); + String message = this.chatMessageQueueForNextFrame.poll(); + if (message == null) + { + // done to prevent potential null pointers + message = ""; + } + MC.sendChatMessage(message); } IProfilerWrapper profiler = MC.getProfiler(); @@ -648,7 +648,10 @@ public class ClientApi } } - // TODO there's probably a better way of handling chat messages - public void showMigrationMessageOnNextFrame() { this.showMigrationMessageNextFrame = true; } + /** + * Queues the given message to appear in chat the next valid frame. + * Useful for queueing up messages that may be triggered before the user has loaded into the world. + */ + public void showChatMessageNextFrame(String chatMessage) { this.chatMessageQueueForNextFrame.add(chatMessage); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 5bafbc87e..a8fbd40c9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -88,6 +88,8 @@ public class FullDataSourceProviderV2 protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); protected final FullDataSourceProviderV1 legacyFileHandler; + protected boolean migrationStartMessageQueued = false; + protected long legacyDeletionCount = -1; protected long migrationCount = -1; @@ -331,7 +333,7 @@ public class FullDataSourceProviderV2 { // this should only be shown once per session but should be shown during // either when the deletion or migration phases start - ClientApi.INSTANCE.showMigrationMessageOnNextFrame(); + this.showMigrationStartMessage(); LOGGER.info("deleting [" + dimensionName + "] - ["+totalDeleteCount+"] unused data sources..."); @@ -382,7 +384,7 @@ public class FullDataSourceProviderV2 ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); if (!legacyDataSourceList.isEmpty()) { - ClientApi.INSTANCE.showMigrationMessageOnNextFrame(); + this.showMigrationStartMessage(); // keep going until every data source has been migrated @@ -453,11 +455,13 @@ public class FullDataSourceProviderV2 if (this.migrationThreadRunning.get()) { LOGGER.info("migration complete for: ["+dimensionName+"]-["+this.saveDir+"]."); + this.showMigrationEndMessage(true); this.migrationCount = 0; } else { LOGGER.info("migration stopped for: ["+dimensionName+"]-["+this.saveDir+"]."); + this.showMigrationEndMessage(false); } } else @@ -472,6 +476,41 @@ public class FullDataSourceProviderV2 public long getTotalMigrationCount() { return this.migrationCount; } + private void showMigrationStartMessage() + { + if (this.migrationStartMessageQueued) + { + return; + } + this.migrationStartMessageQueued = true; + + String dimName = this.level.getLevelWrapper().getDimensionType().getDimensionName(); + ClientApi.INSTANCE.showChatMessageNextFrame( + "Old Distant Horizons data is being migrated for ["+dimName+"]. \n" + + "While migrating LODs may load slowly \n" + + "and DH world gen will be disabled. \n" + + "You can see migration progress in the F3 menu." + ); + } + + private void showMigrationEndMessage(boolean success) + { + String dimName = this.level.getLevelWrapper().getDimensionType().getDimensionName(); + + if (success) + { + ClientApi.INSTANCE.showChatMessageNextFrame("Distant Horizons data migration for ["+dimName+"] completed."); + } + else + { + ClientApi.INSTANCE.showChatMessageNextFrame( + "Distant Horizons data migration for ["+dimName+"] stopped. \n" + + "Some data may not have been migrated." + ); + } + } + + //=======================// // retrieval (world gen) // From 945853d0145c40cd98dc6aae05bee694dc38c240 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 30 Apr 2024 21:53:00 -0500 Subject: [PATCH 11/16] Improve nightly build warning message --- .../seibel/distanthorizons/core/api/internal/ClientApi.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index ef9c06649..ea706e666 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -481,8 +481,8 @@ public class ClientApi this.configOverrideReminderPrinted = true; // remind the user that this is a development build - MC.sendChatMessage("Distant Horizons nightly experimental build version [" + ModInfo.VERSION+"]."); - MC.sendChatMessage("You are running an unsupported version of Distant Horizons!"); + MC.sendChatMessage("Distant Horizons nightly/unstable build, version: [" + ModInfo.VERSION+"]."); + MC.sendChatMessage("Issues may occur with this version."); MC.sendChatMessage("Here be dragons!"); MC.sendChatMessage(""); } From 37eaa2656a5f3d4e6948e57a56cd2b850abbff39 Mon Sep 17 00:00:00 2001 From: cola98765 Date: Wed, 1 May 2024 10:31:43 +0000 Subject: [PATCH 12/16] Update file FullDataToRenderDataTransformer.java --- .../transformers/FullDataToRenderDataTransformer.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 2aabb2e71..1ec8b9a44 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -239,8 +239,14 @@ public class FullDataToRenderDataTransformer { if (colorBelowWithAvoidedBlocks) { - //mare sure to not trnasfer alpha if for some reason grass is transparent - colorToApplyToNextBlock = ColorUtil.setAlpha(level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block),255); + int tempColor = level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block); + if (ColorUtil.getAlpha(tempColor) == 0) + { + //make sure to not transfer the color when alpha is 0 + continue; + } + //mare sure to not trnasfer alpha if for some reason grass is semi transparent + colorToApplyToNextBlock = ColorUtil.setAlpha(tempColor,255); skylightToApplyToNextBlock = skyLight; blocklightToApplyToNextBlock = blockLight; } From 950c951c2de9242dc9bfd34b616efbeee7b2e870 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 2 May 2024 17:31:53 -0500 Subject: [PATCH 13/16] minor ConfigBasedLogger cleanup --- .../core/logging/ConfigBasedLogger.java | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java b/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java index 798080895..7d04d0c2f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java @@ -111,29 +111,10 @@ public class ConfigBasedLogger } } - public void error(String str, Object... param) - { - log(Level.ERROR, str, param); - } - - public void warn(String str, Object... param) - { - log(Level.WARN, str, param); - } - - public void info(String str, Object... param) - { - log(Level.INFO, str, param); - } - - public void debug(String str, Object... param) - { - log(Level.DEBUG, str, param); - } - - public void trace(String str, Object... param) - { - log(Level.TRACE, str, param); - } + public void error(String str, Object... param) { this.log(Level.ERROR, str, param); } + public void warn(String str, Object... param) { this.log(Level.WARN, str, param); } + public void info(String str, Object... param) { this.log(Level.INFO, str, param); } + public void debug(String str, Object... param) { this.log(Level.DEBUG, str, param); } + public void trace(String str, Object... param) { this.log(Level.TRACE, str, param); } } From aad095ca1a83bc20e4493835dab68fb018f7c6fb Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 4 May 2024 09:48:35 -0500 Subject: [PATCH 14/16] Fix #670 Remove outdated world gen options from tooltip --- core/src/main/resources/assets/distanthorizons/lang/en_us.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index c1a127453..ddd0b7127 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -321,7 +321,7 @@ "distanthorizons.config.client.advanced.worldGenerator.distantGeneratorMode": "Distance Generator Mode", "distanthorizons.config.client.advanced.worldGenerator.distantGeneratorMode.@tooltip": - "How complicated the generation should be when generating LODs outside the vanilla render distance.\n\n§6§6Fastest:§r Biome only\n§6Best Quality:§r Features (suggested)", + "How complicated the generation should be when generating LODs outside the vanilla render distance.", "distanthorizons.config.client.advanced.worldGenerator.worldGenLightingEngine": "Lighting Engine", "distanthorizons.config.client.advanced.worldGenerator.worldGenLightingEngine.@tooltip": From f34e67e6bb2570cc54dd2640b7ff1df957387321 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 4 May 2024 15:12:26 -0500 Subject: [PATCH 15/16] Fix F3 levels not closing with multiverse --- .../SubDimensionLevelMatcher.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index a07c3176a..a808e5e8b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -208,14 +208,18 @@ public class SubDimensionLevelMatcher implements AutoCloseable for (File testLevelFolder : this.potentialLevelFolders) { LOGGER.info("Testing level folder: [" + LodUtil.shortenString(testLevelFolder.getName(), 8) + "]"); + + FullDataSourceV2 testFullDataSource = null; try { // get the data source to compare against - IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), this.currentClientLevel, testLevelFolder, false); - FullDataSourceV2 testFullDataSource = tempLevel.getFullDataProvider().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); - if (testFullDataSource == null) + try (IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), this.currentClientLevel, testLevelFolder, false)) { - continue; + testFullDataSource = tempLevel.getFullDataProvider().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); + if (testFullDataSource == null) + { + continue; + } } @@ -334,6 +338,13 @@ public class SubDimensionLevelMatcher implements AutoCloseable // for now we are just assuming it is an unrelated file LOGGER.warn("Error checking level: "+e.getMessage(), e); } + finally + { + if (testFullDataSource != null) + { + try { testFullDataSource.close(); } catch (Exception ignore) {} + } + } } From e1ca398b8f24d82d1c5f79a345cc641a2366d680 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 4 May 2024 15:35:29 -0500 Subject: [PATCH 16/16] Fix updating chunk count not clearing on world close --- .../com/seibel/distanthorizons/core/api/internal/SharedApi.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java index c8ffe6de8..f10c551ad 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java @@ -110,6 +110,8 @@ public class SharedApi ThreadPoolUtil.shutdownThreadPools(); DebugRenderer.clearRenderables(); MC_RENDER.clearTargetFrameBuffer(); + // needs to be closed on world shutdown to clear out un-processed chunks + UPDATING_CHUNK_POS_SET.clear(); // recommend that the garbage collector cleans up any objects from the old world and thread pools System.gc();