From 3b600ce800de5eb2a26cad98bb8ba5c9cd62871b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 28 Apr 2024 15:52:08 -0500 Subject: [PATCH] Add corrupt data read handling --- .../fullData/FullDataPointIdMap.java | 25 ++- .../fullData/sources/FullDataSourceV1.java | 7 +- .../fullData/sources/FullDataSourceV2.java | 22 ++- .../transformers/LodDataBuilder.java | 173 +++++++++--------- .../core/file/AbstractDataSourceHandler.java | 18 +- .../FullDataSourceProviderV1.java | 10 +- .../FullDataSourceProviderV2.java | 5 +- .../core/generation/WorldGenerationQueue.java | 6 + .../core/sql/dto/FullDataSourceV2DTO.java | 47 +++-- .../core/util/FullDataPointUtil.java | 55 ++++-- .../util/objects/DataCorruptedException.java | 24 +++ 11 files changed, 262 insertions(+), 130 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index ce362a4e6..61a29f3e5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -259,9 +260,14 @@ public class FullDataPointIdMap } /** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */ - public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { int entityCount = inputStream.readInt(); + if (entityCount < 0) + { + throw new DataCorruptedException("FullDataPointIdMap deserialize entry count should have a number greater than or equal to 0, returned value ["+entityCount+"]."); + } + // only used when debugging HashMap dataPointEntryBySerialization = new HashMap<>(); @@ -269,6 +275,13 @@ public class FullDataPointIdMap FullDataPointIdMap newMap = new FullDataPointIdMap(pos); for (int i = 0; i < entityCount; i++) { + // necessary to prevent issues with deserializing objects after the level has been closed + if (Thread.interrupted()) + { + throw new InterruptedException(FullDataPointIdMap.class.getSimpleName() + " task interrupted."); + } + + String entryString = inputStream.readUTF(); Entry newEntry = Entry.deserialize(entryString, levelWrapper); newMap.entryList.add(newEntry); @@ -457,18 +470,12 @@ public class FullDataPointIdMap public String serialize() { return this.biome.getSerialString() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString(); } - public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, DataCorruptedException { String[] stringArray = str.split(BLOCK_STATE_SEPARATOR_STRING); if (stringArray.length != 2) { - throw new IOException("Failed to deserialize BiomeBlockStateEntry"); - } - - // necessary to prevent issues with deserializing objects after the level has been closed - if (Thread.interrupted()) - { - throw new InterruptedException(FullDataPointIdMap.class.getSimpleName() + " task interrupted."); + throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry"); } IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapper(stringArray[0], levelWrapper); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 37cdf0452..4f782837b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; @@ -152,7 +153,7 @@ public class FullDataSourceV1 implements IDataSource * Clears and then overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an existing {@link FullDataSourceV1} and can be used in place of a constructor to reuse an existing {@link FullDataSourceV1} object. */ - public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException, DataCorruptedException { // clear/overwrite the old data this.resizeDataStructuresForRepopulation(dto.pos); @@ -166,7 +167,7 @@ public class FullDataSourceV1 implements IDataSource * Overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an empty {@link FullDataSourceV1} and functions similar to a constructor. */ - public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException, DataCorruptedException { FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level); this.setSourceSummaryData(summaryData); @@ -361,7 +362,7 @@ public class FullDataSourceV1 implements IDataSource outputStream.writeInt(DATA_GUARD_BYTE); this.mapping.serialize(outputStream); } - public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { int guardByte = inputStream.readInt(); if (guardByte != DATA_GUARD_BYTE) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 8846ba085..6db6b6411 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -32,6 +32,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; import it.unimi.dsi.fastutil.longs.LongArrayList; @@ -616,7 +617,16 @@ public class FullDataSourceV2 implements IDataSource { if (height != 0) { - newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + try + { + long datapoint = FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight); + newColumnList.add(datapoint); + } + catch (DataCorruptedException e) + { + // shouldn't happen, (especially if validation is disabled) but just in case + LOGGER.warn("Skipping corrupt datapoint for pos "+inputDataSource.pos+" at relative position ["+x+","+z+"] with data: ID["+lastId+"], Height["+height+"], minY["+minY+"], lastBlockLight["+lastBlockLight+"], lastSkyLight["+lastSkyLight+"]."); + } } lastId = id; @@ -630,7 +640,15 @@ public class FullDataSourceV2 implements IDataSource // add the last slice if present if (height != 0) { - newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + try + { + newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + } + catch (DataCorruptedException e) + { + // shouldn't happen, (especially if validation is disabled) but just in case + LOGGER.warn("Skipping corrupt datapoint for pos "+inputDataSource.pos+" at relative position ["+x+","+z+"] with data: ID["+lastId+"], Height["+height+"], minY["+minY+"], lastBlockLight["+lastBlockLight+"], lastSkyLight["+lastSkyLight+"]."); + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 9437ec217..bc6da58f6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; @@ -133,98 +134,106 @@ public class LodDataBuilder EDhApiWorldCompressionMode worldCompressionMode = Config.Client.Advanced.LodBuilding.worldCompression.get(); boolean ignoreHiddenBlocks = (worldCompressionMode != EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS); - int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); - for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) + try { - for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++) + int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); + for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) { - LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); - int lastY = chunkWrapper.getMaxBuildHeight(); - IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ); - IBlockStateWrapper blockState = AIR; - int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - - - byte blockLight; - byte skyLight; - if (lastY < chunkWrapper.getMaxBuildHeight()) + for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++) { - // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting - blockLight = (byte) chunkWrapper.getBlockLight(relBlockX, lastY + 1, relBlockZ); - skyLight = (byte) chunkWrapper.getSkyLight(relBlockX, lastY + 1, relBlockZ); - } - else - { - //we are at the height limit. There are no torches here, and sky is not obscured. - blockLight = 0; - skyLight = 15; - } - - - // determine the starting Y Pos - int y = chunkWrapper.getLightBlockingHeightMapValue(relBlockX,relBlockZ); - // go up until we reach open air or the world limit - IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); - while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight()) - { - try - { - // This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light. - // Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks. - y++; - topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); - } - catch (Exception e) - { - if (!getTopErrorLogged) - { - LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + relBlockX + "," + y + "," + relBlockZ + "] error: " + e.getMessage(), e); - getTopErrorLogged = true; - } - - y--; - break; - } - } - - - for (; y >= minBuildHeight; y--) - { - IBiomeWrapper newBiome = chunkWrapper.getBiome(relBlockX, y, relBlockZ); - IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); - byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ); - byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ); + LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); + int lastY = chunkWrapper.getMaxBuildHeight(); + IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ); + IBlockStateWrapper blockState = AIR; + int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - // save the biome/block change - if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) + + byte blockLight; + byte skyLight; + if (lastY < chunkWrapper.getMaxBuildHeight()) { - // if we ignore hidden blocks, don't save this biome/block change - // wait until the block is visible and then save the new datapoint - if (!ignoreHiddenBlocks - // if the last block is air, this block will always be visible - || blockState.isAir() - // check if this block is visible from any direction - || blockVisible(chunkWrapper, relBlockX, y, relBlockZ)) + // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting + blockLight = (byte) chunkWrapper.getBlockLight(relBlockX, lastY + 1, relBlockZ); + skyLight = (byte) chunkWrapper.getSkyLight(relBlockX, lastY + 1, relBlockZ); + } + else + { + //we are at the height limit. There are no torches here, and sky is not obscured. + blockLight = 0; + skyLight = 15; + } + + + // determine the starting Y Pos + int y = chunkWrapper.getLightBlockingHeightMapValue(relBlockX, relBlockZ); + // go up until we reach open air or the world limit + IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); + while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight()) + { + try { - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - biome = newBiome; - blockState = newBlockState; - mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - blockLight = newBlockLight; - skyLight = newSkyLight; - lastY = y; + // This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light. + // Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks. + y++; + topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); + } + catch (Exception e) + { + if (!getTopErrorLogged) + { + LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + relBlockX + "," + y + "," + relBlockZ + "] error: " + e.getMessage(), e); + getTopErrorLogged = true; + } + + y--; + break; } } + + + for (; y >= minBuildHeight; y--) + { + IBiomeWrapper newBiome = chunkWrapper.getBiome(relBlockX, y, relBlockZ); + IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); + byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ); + byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ); + + // save the biome/block change + if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) + { + // if we ignore hidden blocks, don't save this biome/block change + // wait until the block is visible and then save the new datapoint + if (!ignoreHiddenBlocks + // if the last block is air, this block will always be visible + || blockState.isAir() + // check if this block is visible from any direction + || blockVisible(chunkWrapper, relBlockX, y, relBlockZ)) + { + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + biome = newBiome; + blockState = newBlockState; + mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); + blockLight = newBlockLight; + skyLight = newSkyLight; + lastY = y; + } + } + } + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + + dataSource.setSingleColumn(longs, + relBlockX + chunkOffsetX, + relBlockZ + chunkOffsetZ, + EDhApiWorldGenerationStep.LIGHT, + worldCompressionMode); } - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - - dataSource.setSingleColumn(longs, - relBlockX + chunkOffsetX, - relBlockZ + chunkOffsetZ, - EDhApiWorldGenerationStep.LIGHT, - worldCompressionMode); } } + catch (DataCorruptedException e) + { + LOGGER.error("Unable to convert chunk at pos ["+chunkWrapper.getChunkPos()+"] to an LOD. Error: "+e.getMessage(), e); + return null; + } LodUtil.assertTrue(!dataSource.isEmpty); return dataSource; @@ -292,7 +301,7 @@ public class LodDataBuilder /** @throws ClassCastException if an API user returns the wrong object type(s) */ - public static FullDataSourceV2 createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException + public static FullDataSourceV2 createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException, DataCorruptedException { FullDataSourceV2 accessor = FullDataSourceV2.createEmpty(new DhSectionPos(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ))); for (int relZ = 0; relZ < LodUtil.CHUNK_WIDTH; relZ++) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index b913e16c7..c89168b75 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -8,6 +8,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; @@ -16,6 +17,7 @@ import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; +import java.io.StreamCorruptedException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; @@ -95,7 +97,7 @@ public abstract class AbstractDataSourceHandler /** When this is called the parent folders should be created */ protected abstract TRepo createRepo(); - protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException; + protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException, DataCorruptedException; protected abstract TDTO createDtoFromDataSource(TDataSource dataSource); protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); @@ -145,8 +147,18 @@ public abstract class AbstractDataSourceHandler TDTO dto = this.repo.getByKey(pos); if (dto != null) { - // load from database - dataSource = this.createDataSourceFromDto(dto); + try + { + // load from database + dataSource = this.createDataSourceFromDto(dto); + } + catch (DataCorruptedException e) + { + // stack trace not included since a lot of corrupt data would cause the log to get quite messy, + // and it should be fairly easy to see what the problem was from the message + LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage()); + this.repo.deleteWithKey(pos); + } } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index fde42a6a7..1ee291fcc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -7,6 +7,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV1Repo; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -72,7 +73,7 @@ public class FullDataSourceProviderV1 } } - protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException + protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException, DataCorruptedException { FullDataSourceV1 dataSource = FullDataSourceV1.createEmpty(dto.pos); dataSource.populateFromStream(dto, dto.getInputStream(), this.level); @@ -128,6 +129,13 @@ public class FullDataSourceProviderV1 } } catch (InterruptedException ignore) { } + catch (DataCorruptedException e) + { + // stack trace not included since a lot of corrupt data would cause the log to get quite messy, + // and it should be fairly easy to see what the problem was from the message + LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage()); + this.repo.deleteWithKey(pos); + } catch (IOException e) { LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 795ffb15c..5bafbc87e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.Logger; @@ -165,7 +166,7 @@ public class FullDataSourceProviderV2 } @Override - protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException + protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException { return dto.createPooledDataSource(this.level.getLevelWrapper()); } @Override @@ -263,7 +264,7 @@ public class FullDataSourceProviderV2 } catch (Exception e) { - LOGGER.error("issue in update for parent pos: " + parentUpdatePos); + LOGGER.error("issue in update for parent pos: " + parentUpdatePos+ " Error: "+e.getMessage(), e); } finally { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 04c485bf2..da1595893 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -36,6 +36,7 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException; import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; @@ -469,6 +470,11 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb FullDataSourceV2 dataSource = LodDataBuilder.createFromApiChunkData(dataPoints); chunkDataConsumer.accept(dataSource); } + catch (DataCorruptedException e) + { + LOGGER.error("World generator returned a corrupt chunk. Error: [" + e.getMessage() + "]. World generator disabled.", e); + Config.Client.Advanced.WorldGenerator.enableDistantGeneration.set(false); + } catch (ClassCastException e) { LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index e59007129..bdab7dd0d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -25,21 +25,24 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.jetbrains.annotations.NotNull; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; +import java.io.*; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; /** handles storing {@link FullDataSourceV2}'s in the database. */ public class FullDataSourceV2DTO implements IBaseDTO { + public static final boolean VALIDATE_INPUT_DATAPOINTS = true; + + public DhSectionPos pos; public int levelMinY; @@ -118,23 +121,23 @@ public class FullDataSourceV2DTO implements IBaseDTO // data source population // //========================// - public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(this.pos, false); return this.populateDataSource(dataSource, levelWrapper); } - public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } /** * May be missing one or more data fields.
* Designed to be used without access to Minecraft or any supporting objects. */ - public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException + public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, true); } - private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException + private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException { if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) { @@ -212,7 +215,7 @@ public class FullDataSourceV2DTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); @@ -225,12 +228,21 @@ public class FullDataSourceV2DTO implements IBaseDTO { // read the column length short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later + if (dataColumnLength < 0) + { + throw new DataCorruptedException("Read DataSource Blob data at index ["+xz+"], column length ["+dataColumnLength+"] should be greater than zero."); + } + LongArrayList dataColumn = new LongArrayList(new long[dataColumnLength]); // read column data (will be skipped if no data was present) for (int y = 0; y < dataColumnLength; y++) { long dataPoint = compressedIn.readLong(); + if (VALIDATE_INPUT_DATAPOINTS) + { + FullDataPointUtil.validateDatapoint(dataPoint); + } dataColumn.set(y, dataPoint); } @@ -254,15 +266,22 @@ public class FullDataSourceV2DTO implements IBaseDTO return byteArrayOutputStream.toByteArray(); } - private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); - byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; - compressedIn.readFully(columnGenStepByteArray); - - return columnGenStepByteArray; + try + { + byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + compressedIn.readFully(columnGenStepByteArray); + + return columnGenStepByteArray; + } + catch (EOFException e) + { + throw new DataCorruptedException(e); + } } @@ -302,7 +321,7 @@ public class FullDataSourceV2DTO implements IBaseDTO return byteArrayOutputStream.toByteArray(); } - private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index b63c6f0e7..2b30f66c2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -2,6 +2,7 @@ package com.seibel.distanthorizons.core.util; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.coreapi.ModInfo; import org.jetbrains.annotations.Contract; @@ -72,23 +73,11 @@ public class FullDataPointUtil * creates a new datapoint with the given values * @param relMinY relative to the minimum level Y value */ - public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) + public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) throws DataCorruptedException { if (RUN_VALIDATION) { - // assertions are inside if-blocks to prevent unnecessary string concatenations - if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with y[" + relMinY + "] out of range!"); - } - if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with height[" + height + "] out of range!"); - } - if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with y+depth[" + (relMinY + height) + "] out of range!"); - } + validateData(id, height, relMinY, blockLight, skyLight); } @@ -116,6 +105,44 @@ public class FullDataPointUtil return data; } + public static void validateDatapoint(long datapoint) throws DataCorruptedException { validateData(getId(datapoint), getHeight(datapoint), getBottomY(datapoint), (byte)getBlockLight(datapoint), (byte)getSkyLight(datapoint)); } + /** + * Throws {@link DataCorruptedException} if any of the given values are outside + * their expected range. + */ + public static void validateData(int id, int height, int relMinY, byte blockLight, byte skyLight) throws DataCorruptedException + { + // ID + if (id < 0) + { + throw new DataCorruptedException("Full datapoint ID [" + relMinY + "] must be greater than zero."); + } + + // height + if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + throw new DataCorruptedException("Full datapoint relative min y [" + relMinY + "] must be in the range [0 - "+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"] (inclusive)."); + } + if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + throw new DataCorruptedException("Full datapoint height [" + height + "] must be in the range [1 - "+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"] (inclusive)."); + } + if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + throw new DataCorruptedException("Full datapoint y+depth [" + (relMinY + height) + "] is higher than the maximum world Y height ["+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"]."); + } + + // lighting + if (blockLight < LodUtil.MIN_MC_LIGHT || blockLight > LodUtil.MAX_MC_LIGHT) + { + throw new DataCorruptedException("Full datapoint block light [" + blockLight + "] must be in the range ["+LodUtil.MIN_MC_LIGHT+" - "+LodUtil.MAX_MC_LIGHT+"] (inclusive)."); + } + if (skyLight < LodUtil.MIN_MC_LIGHT || skyLight > LodUtil.MAX_MC_LIGHT) + { + throw new DataCorruptedException("Full datapoint sky light [" + skyLight + "] must be in the range ["+LodUtil.MIN_MC_LIGHT+" - "+LodUtil.MAX_MC_LIGHT+"] (inclusive)."); + } + } + /** Returns the BlockState/Biome pair ID used to identify this LOD's color */ public static int getId(long data) { return (int) (data & ID_MASK); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java new file mode 100644 index 000000000..3195bb320 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java @@ -0,0 +1,24 @@ +package com.seibel.distanthorizons.core.util.objects; + +/** + * Thrown when a DH handled resource or datasource isn't in the + * correct format.

+ * + * IE: a blocklight with the value -4 when it should be between 0 and 15. + */ +public class DataCorruptedException extends Exception +{ + /** replaces this exception's stack trace with the incoming one */ + public DataCorruptedException(Exception e) + { + super(e.getMessage()); + this.setStackTrace(e.getStackTrace()); + this.addSuppressed(e); + } + + public DataCorruptedException(String message) + { + super(message); + } + +}