wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add corrupt data read handling

Browse Source
This commit is contained in:
James Seibel
2024-04-28 15:52:08 -05:00
parent 7f874b4dc5
commit 3b600ce800
11 changed files with 262 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java
+16 -9
View File
@@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
@@ -259,9 +260,14 @@ public class FullDataPointIdMap
}
/** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */
public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException
public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
{
int entityCount = inputStream.readInt();
if (entityCount < 0)
{
throw new DataCorruptedException("FullDataPointIdMap deserialize entry count should have a number greater than or equal to 0, returned value ["+entityCount+"].");
}
// only used when debugging
HashMap<String, FullDataPointIdMap.Entry> dataPointEntryBySerialization = new HashMap<>();
@@ -269,6 +275,13 @@ public class FullDataPointIdMap
FullDataPointIdMap newMap = new FullDataPointIdMap(pos);
for (int i = 0; i < entityCount; i++)
{
// necessary to prevent issues with deserializing objects after the level has been closed
if (Thread.interrupted())
{
throw new InterruptedException(FullDataPointIdMap.class.getSimpleName() + " task interrupted.");
}
String entryString = inputStream.readUTF();
Entry newEntry = Entry.deserialize(entryString, levelWrapper);
newMap.entryList.add(newEntry);
@@ -457,18 +470,12 @@ public class FullDataPointIdMap
public String serialize() { return this.biome.getSerialString() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString(); }
public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, InterruptedException
public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, DataCorruptedException
{
String[] stringArray = str.split(BLOCK_STATE_SEPARATOR_STRING);
if (stringArray.length != 2)
{
throw new IOException("Failed to deserialize BiomeBlockStateEntry");
}
// necessary to prevent issues with deserializing objects after the level has been closed
if (Thread.interrupted())
{
throw new InterruptedException(FullDataPointIdMap.class.getSimpleName() + " task interrupted.");
throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry");
}
IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapper(stringArray[0], levelWrapper);
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java
+4 -3
View File
@@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
@@ -152,7 +153,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
* Clears and then overwrites any data in this object with the data from the given file and stream.
* This is expected to be used with an existing {@link FullDataSourceV1} and can be used in place of a constructor to reuse an existing {@link FullDataSourceV1} object.
*/
public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException, DataCorruptedException
{
// clear/overwrite the old data
this.resizeDataStructuresForRepopulation(dto.pos);
@@ -166,7 +167,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
* Overwrites any data in this object with the data from the given file and stream.
* This is expected to be used with an empty {@link FullDataSourceV1} and functions similar to a constructor.
*/
public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException, DataCorruptedException
{
FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level);
this.setSourceSummaryData(summaryData);
@@ -361,7 +362,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
outputStream.writeInt(DATA_GUARD_BYTE);
this.mapping.serialize(outputStream);
}
public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException
public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
{
int guardByte = inputStream.readInt();
if (guardByte != DATA_GUARD_BYTE)
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+20 -2
View File
@@ -32,6 +32,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
import com.seibel.distanthorizons.coreapi.ModInfo;
import it.unimi.dsi.fastutil.longs.LongArrayList;
@@ -616,7 +617,16 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
{
if (height != 0)
{
newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight));
try
{
long datapoint = FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight);
newColumnList.add(datapoint);
}
catch (DataCorruptedException e)
{
// shouldn't happen, (especially if validation is disabled) but just in case
LOGGER.warn("Skipping corrupt datapoint for pos "+inputDataSource.pos+" at relative position ["+x+","+z+"] with data: ID["+lastId+"], Height["+height+"], minY["+minY+"], lastBlockLight["+lastBlockLight+"], lastSkyLight["+lastSkyLight+"].");
}
}
lastId = id;
@@ -630,7 +640,15 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// add the last slice if present
if (height != 0)
{
newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight));
try
{
newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight));
}
catch (DataCorruptedException e)
{
// shouldn't happen, (especially if validation is disabled) but just in case
LOGGER.warn("Skipping corrupt datapoint for pos "+inputDataSource.pos+" at relative position ["+x+","+z+"] with data: ID["+lastId+"], Height["+height+"], minY["+minY+"], lastBlockLight["+lastBlockLight+"], lastSkyLight["+lastSkyLight+"].");
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java
+91 -82
View File
@@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
@@ -133,98 +134,106 @@ public class LodDataBuilder
EDhApiWorldCompressionMode worldCompressionMode = Config.Client.Advanced.LodBuilding.worldCompression.get();
boolean ignoreHiddenBlocks = (worldCompressionMode != EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS);
int minBuildHeight = chunkWrapper.getMinNonEmptyHeight();
for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++)
try
{
for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++)
int minBuildHeight = chunkWrapper.getMinNonEmptyHeight();
for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++)
{
LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4);
int lastY = chunkWrapper.getMaxBuildHeight();
IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ);
IBlockStateWrapper blockState = AIR;
int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState);
byte blockLight;
byte skyLight;
if (lastY < chunkWrapper.getMaxBuildHeight())
for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++)
{
// FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting
blockLight = (byte) chunkWrapper.getBlockLight(relBlockX, lastY + 1, relBlockZ);
skyLight = (byte) chunkWrapper.getSkyLight(relBlockX, lastY + 1, relBlockZ);
}
else
{
//we are at the height limit. There are no torches here, and sky is not obscured.
blockLight = 0;
skyLight = 15;
}
// determine the starting Y Pos
int y = chunkWrapper.getLightBlockingHeightMapValue(relBlockX,relBlockZ);
// go up until we reach open air or the world limit
IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ);
while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight())
{
try
{
// This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light.
// Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks.
y++;
topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ);
}
catch (Exception e)
{
if (!getTopErrorLogged)
{
LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + relBlockX + "," + y + "," + relBlockZ + "] error: " + e.getMessage(), e);
getTopErrorLogged = true;
}
y--;
break;
}
}
for (; y >= minBuildHeight; y--)
{
IBiomeWrapper newBiome = chunkWrapper.getBiome(relBlockX, y, relBlockZ);
IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ);
byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ);
byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ);
LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4);
int lastY = chunkWrapper.getMaxBuildHeight();
IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ);
IBlockStateWrapper blockState = AIR;
int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState);
// save the biome/block change
if (!newBiome.equals(biome) || !newBlockState.equals(blockState))
byte blockLight;
byte skyLight;
if (lastY < chunkWrapper.getMaxBuildHeight())
{
// if we ignore hidden blocks, don't save this biome/block change
// wait until the block is visible and then save the new datapoint
if (!ignoreHiddenBlocks
// if the last block is air, this block will always be visible
|| blockState.isAir()
// check if this block is visible from any direction
|| blockVisible(chunkWrapper, relBlockX, y, relBlockZ))
// FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting
blockLight = (byte) chunkWrapper.getBlockLight(relBlockX, lastY + 1, relBlockZ);
skyLight = (byte) chunkWrapper.getSkyLight(relBlockX, lastY + 1, relBlockZ);
}
else
{
//we are at the height limit. There are no torches here, and sky is not obscured.
blockLight = 0;
skyLight = 15;
}
// determine the starting Y Pos
int y = chunkWrapper.getLightBlockingHeightMapValue(relBlockX, relBlockZ);
// go up until we reach open air or the world limit
IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ);
while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight())
{
try
{
longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight));
biome = newBiome;
blockState = newBlockState;
mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState);
blockLight = newBlockLight;
skyLight = newSkyLight;
lastY = y;
// This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light.
// Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks.
y++;
topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ);
}
catch (Exception e)
{
if (!getTopErrorLogged)
{
LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + relBlockX + "," + y + "," + relBlockZ + "] error: " + e.getMessage(), e);
getTopErrorLogged = true;
}
y--;
break;
}
}
for (; y >= minBuildHeight; y--)
{
IBiomeWrapper newBiome = chunkWrapper.getBiome(relBlockX, y, relBlockZ);
IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ);
byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ);
byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ);
// save the biome/block change
if (!newBiome.equals(biome) || !newBlockState.equals(blockState))
{
// if we ignore hidden blocks, don't save this biome/block change
// wait until the block is visible and then save the new datapoint
if (!ignoreHiddenBlocks
// if the last block is air, this block will always be visible
|| blockState.isAir()
// check if this block is visible from any direction
|| blockVisible(chunkWrapper, relBlockX, y, relBlockZ))
{
longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight));
biome = newBiome;
blockState = newBlockState;
mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState);
blockLight = newBlockLight;
skyLight = newSkyLight;
lastY = y;
}
}
}
longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight));
dataSource.setSingleColumn(longs,
relBlockX + chunkOffsetX,
relBlockZ + chunkOffsetZ,
EDhApiWorldGenerationStep.LIGHT,
worldCompressionMode);
}
longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight));
dataSource.setSingleColumn(longs,
relBlockX + chunkOffsetX,
relBlockZ + chunkOffsetZ,
EDhApiWorldGenerationStep.LIGHT,
worldCompressionMode);
}
}
catch (DataCorruptedException e)
{
LOGGER.error("Unable to convert chunk at pos ["+chunkWrapper.getChunkPos()+"] to an LOD. Error: "+e.getMessage(), e);
return null;
}
LodUtil.assertTrue(!dataSource.isEmpty);
return dataSource;
@@ -292,7 +301,7 @@ public class LodDataBuilder
/** @throws ClassCastException if an API user returns the wrong object type(s) */
public static FullDataSourceV2 createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException
public static FullDataSourceV2 createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException, DataCorruptedException
{
FullDataSourceV2 accessor = FullDataSourceV2.createEmpty(new DhSectionPos(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ)));
for (int relZ = 0; relZ < LodUtil.CHUNK_WIDTH; relZ++)
core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java
+15 -3
View File
@@ -8,6 +8,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import com.seibel.distanthorizons.core.sql.dto.IBaseDTO;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import org.apache.logging.log4j.Logger;
@@ -16,6 +17,7 @@ import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.io.StreamCorruptedException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -95,7 +97,7 @@ public abstract class AbstractDataSourceHandler
/** When this is called the parent folders should be created */
protected abstract TRepo createRepo();
protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException;
protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException, DataCorruptedException;
protected abstract TDTO createDtoFromDataSource(TDataSource dataSource);
protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos);
@@ -145,8 +147,18 @@ public abstract class AbstractDataSourceHandler
TDTO dto = this.repo.getByKey(pos);
if (dto != null)
{
// load from database
dataSource = this.createDataSourceFromDto(dto);
try
{
// load from database
dataSource = this.createDataSourceFromDto(dto);
}
catch (DataCorruptedException e)
{
// stack trace not included since a lot of corrupt data would cause the log to get quite messy,
// and it should be fairly easy to see what the problem was from the message
LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage());
this.repo.deleteWithKey(pos);
}
}
else
{
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java
+9 -1
View File
@@ -7,6 +7,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV1Repo;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
@@ -72,7 +73,7 @@ public class FullDataSourceProviderV1<TDhLevel extends IDhLevel>
}
}
protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException
protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException, DataCorruptedException
{
FullDataSourceV1 dataSource = FullDataSourceV1.createEmpty(dto.pos);
dataSource.populateFromStream(dto, dto.getInputStream(), this.level);
@@ -128,6 +129,13 @@ public class FullDataSourceProviderV1<TDhLevel extends IDhLevel>
}
}
catch (InterruptedException ignore) { }
catch (DataCorruptedException e)
{
// stack trace not included since a lot of corrupt data would cause the log to get quite messy,
// and it should be fairly easy to see what the problem was from the message
LOGGER.warn("Corrupted data found at pos "+pos+". Data at position will be deleted so it can be re-generated and to prevent future issues. Error: "+e.getMessage());
this.repo.deleteWithKey(pos);
}
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e);
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java
+3 -2
View File
@@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import org.apache.logging.log4j.Logger;
@@ -165,7 +166,7 @@ public class FullDataSourceProviderV2
}
@Override
protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException
protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException
{ return dto.createPooledDataSource(this.level.getLevelWrapper()); }
@Override
@@ -263,7 +264,7 @@ public class FullDataSourceProviderV2
}
catch (Exception e)
{
LOGGER.error("issue in update for parent pos: " + parentUpdatePos);
LOGGER.error("issue in update for parent pos: " + parentUpdatePos+ " Error: "+e.getMessage(), e);
}
finally
{
core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java
+6
View File
@@ -36,6 +36,7 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
@@ -469,6 +470,11 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
FullDataSourceV2 dataSource = LodDataBuilder.createFromApiChunkData(dataPoints);
chunkDataConsumer.accept(dataSource);
}
catch (DataCorruptedException e)
{
LOGGER.error("World generator returned a corrupt chunk. Error: [" + e.getMessage() + "]. World generator disabled.", e);
Config.Client.Advanced.WorldGenerator.enableDistantGeneration.set(false);
}
catch (ClassCastException e)
{
LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e);
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+33 -14
View File
@@ -25,21 +25,24 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.jetbrains.annotations.NotNull;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.*;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
/** handles storing {@link FullDataSourceV2}'s in the database. */
public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
{
public static final boolean VALIDATE_INPUT_DATAPOINTS = true;
public DhSectionPos pos;
public int levelMinY;
@@ -118,23 +121,23 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
// data source population //
//========================//
public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException
public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
{
FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(this.pos, false);
return this.populateDataSource(dataSource, levelWrapper);
}
public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException
public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
{ return this.internalPopulateDataSource(dataSource, levelWrapper, false); }
/**
* May be missing one or more data fields. <br>
* Designed to be used without access to Minecraft or any supporting objects.
*/
public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException
public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException
{ return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, true); }
private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException
private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException
{
if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion)
{
@@ -212,7 +215,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray());
}
private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray);
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
@@ -225,12 +228,21 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
{
// read the column length
short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later
if (dataColumnLength < 0)
{
throw new DataCorruptedException("Read DataSource Blob data at index ["+xz+"], column length ["+dataColumnLength+"] should be greater than zero.");
}
LongArrayList dataColumn = new LongArrayList(new long[dataColumnLength]);
// read column data (will be skipped if no data was present)
for (int y = 0; y < dataColumnLength; y++)
{
long dataPoint = compressedIn.readLong();
if (VALIDATE_INPUT_DATAPOINTS)
{
FullDataPointUtil.validateDatapoint(dataPoint);
}
dataColumn.set(y, dataPoint);
}
@@ -254,15 +266,22 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
return byteArrayOutputStream.toByteArray();
}
private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException
private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray);
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
compressedIn.readFully(columnGenStepByteArray);
return columnGenStepByteArray;
try
{
byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
compressedIn.readFully(columnGenStepByteArray);
return columnGenStepByteArray;
}
catch (EOFException e)
{
throw new DataCorruptedException(e);
}
}
@@ -302,7 +321,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
return byteArrayOutputStream.toByteArray();
}
private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException
private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray);
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java
+41 -14
View File
@@ -2,6 +2,7 @@ package com.seibel.distanthorizons.core.util;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.coreapi.ModInfo;
import org.jetbrains.annotations.Contract;
@@ -72,23 +73,11 @@ public class FullDataPointUtil
* creates a new datapoint with the given values
* @param relMinY relative to the minimum level Y value
*/
public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight)
public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) throws DataCorruptedException
{
if (RUN_VALIDATION)
{
// assertions are inside if-blocks to prevent unnecessary string concatenations
if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE)
{
LodUtil.assertNotReach("Trying to create datapoint with y[" + relMinY + "] out of range!");
}
if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE)
{
LodUtil.assertNotReach("Trying to create datapoint with height[" + height + "] out of range!");
}
if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE)
{
LodUtil.assertNotReach("Trying to create datapoint with y+depth[" + (relMinY + height) + "] out of range!");
}
validateData(id, height, relMinY, blockLight, skyLight);
}
@@ -116,6 +105,44 @@ public class FullDataPointUtil
return data;
}
public static void validateDatapoint(long datapoint) throws DataCorruptedException { validateData(getId(datapoint), getHeight(datapoint), getBottomY(datapoint), (byte)getBlockLight(datapoint), (byte)getSkyLight(datapoint)); }
/**
* Throws {@link DataCorruptedException} if any of the given values are outside
* their expected range.
*/
public static void validateData(int id, int height, int relMinY, byte blockLight, byte skyLight) throws DataCorruptedException
{
// ID
if (id < 0)
{
throw new DataCorruptedException("Full datapoint ID [" + relMinY + "] must be greater than zero.");
}
// height
if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE)
{
throw new DataCorruptedException("Full datapoint relative min y [" + relMinY + "] must be in the range [0 - "+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"] (inclusive).");
}
if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE)
{
throw new DataCorruptedException("Full datapoint height [" + height + "] must be in the range [1 - "+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"] (inclusive).");
}
if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE)
{
throw new DataCorruptedException("Full datapoint y+depth [" + (relMinY + height) + "] is higher than the maximum world Y height ["+RenderDataPointUtil.MAX_WORLD_Y_SIZE+"].");
}
// lighting
if (blockLight < LodUtil.MIN_MC_LIGHT || blockLight > LodUtil.MAX_MC_LIGHT)
{
throw new DataCorruptedException("Full datapoint block light [" + blockLight + "] must be in the range ["+LodUtil.MIN_MC_LIGHT+" - "+LodUtil.MAX_MC_LIGHT+"] (inclusive).");
}
if (skyLight < LodUtil.MIN_MC_LIGHT || skyLight > LodUtil.MAX_MC_LIGHT)
{
throw new DataCorruptedException("Full datapoint sky light [" + skyLight + "] must be in the range ["+LodUtil.MIN_MC_LIGHT+" - "+LodUtil.MAX_MC_LIGHT+"] (inclusive).");
}
}
/** Returns the BlockState/Biome pair ID used to identify this LOD's color */
public static int getId(long data) { return (int) (data & ID_MASK); }
core/src/main/java/com/seibel/distanthorizons/core/util/objects/DataCorruptedException.java
+24
View File
@@ -0,0 +1,24 @@
package com.seibel.distanthorizons.core.util.objects;
/**
* Thrown when a DH handled resource or datasource isn't in the
* correct format. <Br><Br>
*
* IE: a blocklight with the value -4 when it should be between 0 and 15.
*/
public class DataCorruptedException extends Exception
{
/** replaces this exception's stack trace with the incoming one */
public DataCorruptedException(Exception e)
{
super(e.getMessage());
this.setStackTrace(e.getStackTrace());
this.addSuppressed(e);
}
public DataCorruptedException(String message)
{
super(message);
}
}