wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Repo and Obj Pool rewrite

Browse Source 
This should provide a significant reduction in garbage generated, reducing GC pressure.
This commit is contained in:
James Seibel
2024-12-20 13:38:44 -06:00
parent 7e0c10a516
commit 0ba030e4aa
46 changed files with 2162 additions and 1024 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java
+1 -1
View File
@@ -382,7 +382,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
//==================//
@Override
public void close() throws Exception
public void close()
{ /* not currently needed */ }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+105 -90
View File
@@ -26,18 +26,16 @@ import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
import com.seibel.distanthorizons.core.file.DataSourcePool;
import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.DhApiTerrainDataPointUtil;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import com.seibel.distanthorizons.core.util.*;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
import com.seibel.distanthorizons.coreapi.ModInfo;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
@@ -53,7 +51,9 @@ import java.util.List;
* @see FullDataPointUtil
* @see FullDataSourceV1
*/
public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSource
public class FullDataSourceV2
extends PhantomArrayListParent
implements IDataSource<IDhLevel>, IDhApiFullDataSource
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
/** useful for debugging, but can slow down update operations quite a bit due to being called so often. */
@@ -72,13 +72,11 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
public static final byte DATA_FORMAT_VERSION = 1;
public static final DataSourcePool<FullDataSourceV2, IDhLevel> DATA_SOURCE_POOL = new DataSourcePool<>(FullDataSourceV2::createEmpty, FullDataSourceV2::prepPooledDataSource);
private int cachedHashCode = 0;
private long pos;
private final long pos;
@Override
public Long getKey() { return this.pos; }
@Override
@@ -98,13 +96,13 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
*
* @see EDhApiWorldGenerationStep
*/
public byte[] columnGenerationSteps;
public final ByteArrayList columnGenerationSteps;
/**
* stores what world compression was used for each column.
*
* @see EDhApiWorldCompressionMode
*/
public byte[] columnWorldCompressionMode;
public final ByteArrayList columnWorldCompressionMode;
/**
* stored x/z, y <br>
@@ -112,12 +110,12 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
* TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately
* there's something in the render data logic that expects this order so we can't change it right now
*/
public LongArrayList[] dataPoints;
public final LongArrayList[] dataPoints;
public boolean isEmpty;
public boolean applyToParent = false;
/** should only be used by methods exposed by the DH API */
/** should only be used by methods exposed via the DH API */
private boolean runApiChunkValidation = false;
@@ -126,34 +124,6 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
// constructors //
//==============//
public static FullDataSourceV2 createEmpty(long pos) { return new FullDataSourceV2(pos); }
private FullDataSourceV2(long pos)
{
this.pos = pos;
this.dataPoints = new LongArrayList[WIDTH * WIDTH];
this.mapping = new FullDataPointIdMap(pos);
this.isEmpty = true;
// doesn't need to be populated since nothing has been generated yet
// the default value of all 0's is adequate
this.columnGenerationSteps = new byte[WIDTH * WIDTH];
this.columnWorldCompressionMode = new byte[WIDTH * WIDTH];
}
public static FullDataSourceV2 createWithData(long pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep, byte[] columnWorldCompressionMode) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep, columnWorldCompressionMode); }
private FullDataSourceV2(long pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps, byte[] columnWorldCompressionMode)
{
LodUtil.assertTrue(data.length == WIDTH * WIDTH);
this.pos = pos;
this.dataPoints = data;
this.mapping = mapping;
this.isEmpty = false;
this.columnGenerationSteps = columnGenerationSteps;
this.columnWorldCompressionMode = columnWorldCompressionMode;
}
public static FullDataSourceV2 createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createFromChunk(chunkWrapper); }
public static FullDataSourceV2 createFromLegacyDataSourceV1(FullDataSourceV1 legacyData)
@@ -225,6 +195,84 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
return fullDataSource;
}
public static FullDataSourceV2 createEmpty(long pos)
{
return new FullDataSourceV2(
pos, new FullDataPointIdMap(pos),
// data points, genSteps, and columnCompression are all null since
// nothing has been generated yet.
// Using the default value of all 0's is adequate
null,
null, null,
true);
}
public static FullDataSourceV2 createWithData(long pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep, byte[] columnWorldCompressionMode)
{ return new FullDataSourceV2(pos, mapping, data, columnGenerationStep, columnWorldCompressionMode, false); }
private FullDataSourceV2(
long pos,
FullDataPointIdMap mapping, @Nullable LongArrayList[] data,
@Nullable byte[] columnGenerationSteps, @Nullable byte[] columnWorldCompressionMode,
boolean empty)
{
super(2, 0, WIDTH * WIDTH);
LodUtil.assertTrue(data == null || data.length == WIDTH * WIDTH);
this.pos = pos;
this.mapping = mapping;
this.isEmpty = empty;
// pooled data arrays
this.dataPoints = new LongArrayList[WIDTH * WIDTH];
for (int i = 0; i < WIDTH * WIDTH; i++)
{
// size defaulting to 0 since we don't know how many datapoints
// will be in this column yet
this.dataPoints[i] = this.pooledArraysCheckout.getLongArray(i, 0);
}
// use incoming data if present
if (data != null)
{
for (int i = 0; i < WIDTH * WIDTH; i++)
{
this.dataPoints[i].addAll(data[i]);
}
}
// pooled generation step array
this.columnGenerationSteps = this.pooledArraysCheckout.getByteArray(0);
if (columnGenerationSteps != null)
{
this.columnGenerationSteps.addElements(0, columnGenerationSteps);
}
else
{
ListUtil.clearAndSetSize(this.columnGenerationSteps, WIDTH * WIDTH);
}
// pooled column compression array
this.columnWorldCompressionMode = this.pooledArraysCheckout.getByteArray(1);
if (columnWorldCompressionMode != null)
{
this.columnWorldCompressionMode.addElements(0, columnWorldCompressionMode);
}
else
{
ListUtil.clearAndSetSize(this.columnWorldCompressionMode, WIDTH * WIDTH);
}
// the pooled arrays have all been set,
// the checkout object is no longer needed
this.pooledArraysCheckout = null;
}
//======//
@@ -300,8 +348,8 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
LongArrayList inputDataArray = inputDataSource.dataPoints[index];
if (inputDataArray != null)
{
byte thisGenState = this.columnGenerationSteps[index];
byte inputGenState = inputDataSource.columnGenerationSteps[index];
byte thisGenState = this.columnGenerationSteps.getByte(index);
byte inputGenState = inputDataSource.columnGenerationSteps.getByte(index);
if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value
&& thisGenState <= inputGenState)
@@ -352,9 +400,9 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
}
this.columnGenerationSteps[index] = inputGenState;
this.columnGenerationSteps.set(index, inputGenState);
// always overwrite the compression mode since we're replacing this column
this.columnWorldCompressionMode[index] = inputDataSource.columnWorldCompressionMode[index];
this.columnWorldCompressionMode.set(index, inputDataSource.columnWorldCompressionMode.getByte(index));
this.isEmpty = false;
}
}
@@ -397,12 +445,12 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
// world gen //
byte inputGenStep = determineMinWorldGenStepForTwoByTwoColumn(inputDataSource.columnGenerationSteps, x, z);
this.columnGenerationSteps[recipientIndex] = inputGenStep;
this.columnGenerationSteps.set(recipientIndex, inputGenStep);
// world compression //
byte worldCompressionMode = determineHighestWorldCompressionForTwoByTwoColumn(inputDataSource.columnWorldCompressionMode, x, z);
this.columnWorldCompressionMode[recipientIndex] = worldCompressionMode;
this.columnWorldCompressionMode.set(recipientIndex, worldCompressionMode);
@@ -461,7 +509,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
* The minimum value is used because we don't want to accidentally record that
* something was generated when it wasn't.
*/
private static byte determineMinWorldGenStepForTwoByTwoColumn(byte[] columnGenerationSteps, int relX, int relZ)
private static byte determineMinWorldGenStepForTwoByTwoColumn(ByteArrayList columnGenerationSteps, int relX, int relZ)
{
// TODO merge similar logic with determineHighestWorldCompressionForTwoByTwoColumn
byte minWorldGenStepValue = Byte.MAX_VALUE;
@@ -470,7 +518,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
for (int z = 0; z < 2; z++)
{
int index = relativePosToIndex(x + relX, z + relZ);
byte worldGenStepValue = columnGenerationSteps[index];
byte worldGenStepValue = columnGenerationSteps.getByte(index);
minWorldGenStepValue = (byte) Math.min(minWorldGenStepValue, worldGenStepValue);
}
}
@@ -480,7 +528,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
* The minimum value is used because we don't want to accidentally record that
* something was generated when it wasn't.
*/
private static byte determineHighestWorldCompressionForTwoByTwoColumn(byte[] columnCompressionMode, int relX, int relZ)
private static byte determineHighestWorldCompressionForTwoByTwoColumn(ByteArrayList columnCompressionMode, int relX, int relZ)
{
// TODO merge similar logic with determineMinWorldGenStepForTwoByTwoColumn
byte minWorldGenStepValue = Byte.MIN_VALUE;
@@ -489,7 +537,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
for (int z = 0; z < 2; z++)
{
int index = relativePosToIndex(x + relX, z + relZ);
byte worldGenStepValue = columnCompressionMode[index];
byte worldGenStepValue = columnCompressionMode.getByte(index);
minWorldGenStepValue = (byte) Math.max(minWorldGenStepValue, worldGenStepValue);
}
}
@@ -855,37 +903,6 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
//=========//
// pooling //
//=========//
private static void prepPooledDataSource(long pos, boolean clearData, FullDataSourceV2 dataSource)
{
dataSource.pos = pos;
if (clearData)
{
dataSource.mapping.clear(pos);
for (int i = 0; i < dataSource.dataPoints.length; i++)
{
if (dataSource.dataPoints[i] != null)
{
dataSource.dataPoints[i].clear();
}
}
Arrays.fill(dataSource.columnGenerationSteps, (byte) 0);
Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0);
}
// default to API validation disabled so it's opt-in
// to reduce the chance of performance loss with unnecessary validation
dataSource.setRunApiChunkValidation(false);
}
//=====================//
// setters and getters //
//=====================//
@@ -899,15 +916,15 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ)
{
int index = relativePosToIndex(relX, relZ);
return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]);
return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps.getByte(index));
}
public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep, EDhApiWorldCompressionMode worldCompressionMode)
{
int index = relativePosToIndex(relX, relZ);
this.dataPoints[index] = longArray;
this.columnGenerationSteps[index] = worldGenStep.value;
this.columnWorldCompressionMode[index] = worldCompressionMode.value;
this.columnGenerationSteps.set(index, worldGenStep.value);
this.columnWorldCompressionMode.set(index, worldCompressionMode.value);
if (RUN_UPDATE_DEV_VALIDATION)
@@ -1002,8 +1019,8 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
{
int result = DhSectionPos.hashCode(this.pos);
result = 31 * result + Arrays.deepHashCode(this.dataPoints);
result = 17 * result + Arrays.hashCode(this.columnGenerationSteps);
result = 43 * result + Arrays.hashCode(this.columnWorldCompressionMode);
result = 17 * result + this.columnGenerationSteps.hashCode();
result = 43 * result + this.columnWorldCompressionMode.hashCode();
this.cachedHashCode = result;
}
@@ -1030,8 +1047,6 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSo
}
}
@Override
public void close() throws Exception
{ DATA_SOURCE_POOL.returnPooledDataSource(this); }
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
+14 -48
View File
@@ -22,9 +22,9 @@ package com.seibel.distanthorizons.core.dataObjects.render;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.file.DataSourcePool;
import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.coreapi.ModInfo;
@@ -37,7 +37,6 @@ import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -45,7 +44,9 @@ import java.util.concurrent.atomic.AtomicLong;
*
* @see RenderDataPointUtil
*/
public class ColumnRenderSource implements IDataSource<IDhClientLevel>
public class ColumnRenderSource
extends PhantomArrayListParent
implements IDataSource<IDhClientLevel>
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@@ -54,8 +55,6 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
/** width of this data in columns */
public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); // 64
public static final DataSourcePool<ColumnRenderSource, IDhClientLevel> DATA_SOURCE_POOL = new DataSourcePool<>(ColumnRenderSource::createEmptyRenderSource, null /* data source prep/cleanup needs to be done outside the pool since it requires additional inputs */);
/** will be zero if an empty data source was created */
@@ -77,40 +76,8 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
// constructors //
//==============//
/**
* This is separate from {@link DataSourcePool#getPooledSource(long, boolean)}
* because we need to pass in a couple extra values,
* specifically maxVerticalSize and yOffset.
*/
public static ColumnRenderSource getPooledRenderSource(long pos, int maxVerticalSize, int yOffset, boolean clearData)
{
ColumnRenderSource renderSource = DATA_SOURCE_POOL.getPooledSource(pos);
// set necessary properties
renderSource.pos = pos;
renderSource.verticalDataCount = maxVerticalSize;
renderSource.yOffset = yOffset;
// resize the array if necessary
int dataArraySize = SECTION_SIZE * SECTION_SIZE * maxVerticalSize;
renderSource.renderDataContainer.ensureCapacity(dataArraySize);
while (renderSource.renderDataContainer.size() < dataArraySize)
{
renderSource.renderDataContainer.add(0);
}
if (clearData)
{
Arrays.fill(renderSource.renderDataContainer.elements(), 0);
Arrays.fill(renderSource.debugSourceFlags, null);
}
return renderSource;
}
private static ColumnRenderSource createEmptyRenderSource(long sectionPos) { return new ColumnRenderSource(sectionPos, 0, 0); }
public static ColumnRenderSource createEmpty(long pos, int maxVerticalSize, int yOffset)
{ return new ColumnRenderSource(pos, maxVerticalSize, yOffset); }
/**
* Creates an empty ColumnRenderSource.
*
@@ -119,11 +86,16 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
*/
private ColumnRenderSource(long pos, int maxVerticalSize, int yOffset)
{
this.verticalDataCount = maxVerticalSize;
this.renderDataContainer = new LongArrayList(new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]);
this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE];
super(0, 0, 1);
this.pos = pos;
this.yOffset = yOffset;
this.verticalDataCount = maxVerticalSize;
this.renderDataContainer = this.pooledArraysCheckout.getLongArray(0, SECTION_SIZE * SECTION_SIZE * this.verticalDataCount);
this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE];
}
@@ -327,12 +299,6 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
return stringBuilder.toString();
}
@Override
public void close() throws Exception
{
DATA_SOURCE_POOL.returnPooledDataSource(this);
}
//==============//
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java
+21 -10
View File
@@ -28,13 +28,11 @@ import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArra
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPosMutable;
import com.seibel.distanthorizons.core.util.ColorUtil;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import com.seibel.distanthorizons.core.util.*;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
@@ -43,6 +41,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.util.HashSet;
@@ -64,6 +63,7 @@ public class FullDataToRenderDataTransformer
// public transformer interface //
//==============================//
@Nullable
public static ColumnRenderSource transformFullDataToRenderSource(FullDataSourceV2 fullDataSource, IDhClientLevel level)
{
if (fullDataSource == null)
@@ -109,7 +109,7 @@ public class FullDataToRenderDataTransformer
final ColumnRenderSource columnSource = ColumnRenderSource.getPooledRenderSource(pos, vertSize, level.getMinY(), true);
final ColumnRenderSource columnSource = ColumnRenderSource.createEmpty(pos, vertSize, level.getMinY());
if (fullDataSource.isEmpty)
{
return columnSource;
@@ -162,10 +162,21 @@ public class FullDataToRenderDataTransformer
}
else
{
// expand the ColumnArrayView to fit the new larger max vertical size
ColumnArrayView newColumnArrayView = new ColumnArrayView(new LongArrayList(new long[fullDataLength]), fullDataLength, 0, fullDataLength);
setRenderColumnView(level, fullDataMapping, blockX, blockZ, newColumnArrayView, fullDataColumn);
columnArrayView.changeVerticalSizeFrom(newColumnArrayView);
// new LongArrayList(new long[fullDataLength])
PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(0, 0, 1);
LongArrayList dataArrayList = checkout.getLongArray(0, fullDataLength);
try
{
// expand the ColumnArrayView to fit the new larger max vertical size
ColumnArrayView newColumnArrayView = new ColumnArrayView(dataArrayList, fullDataLength, 0, fullDataLength);
setRenderColumnView(level, fullDataMapping, blockX, blockZ, newColumnArrayView, fullDataColumn);
columnArrayView.changeVerticalSizeFrom(newColumnArrayView);
}
finally
{
PhantomArrayListPool.INSTANCE.returnCheckout(checkout);
}
}
}
private static void setRenderColumnView(
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java
+14 -3
View File
@@ -71,7 +71,7 @@ public class LodDataBuilder
int sectionPosZ = getXOrZSectionPosFromChunkPos(chunkWrapper.getChunkPos().getZ());
long pos = DhSectionPos.encode(DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL, sectionPosX, sectionPosZ);
FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos);
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos);
dataSource.isEmpty = false;
@@ -140,7 +140,18 @@ public class LodDataBuilder
{
for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++)
{
LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4);
LongArrayList longs = dataSource.get(
relBlockX + chunkOffsetX,
relBlockZ + chunkOffsetZ);
if (longs == null)
{
longs = new LongArrayList(chunkWrapper.getHeight() / 4);
}
else
{
longs.clear();
}
int lastY = chunkWrapper.getExclusiveMaxBuildHeight();
IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ);
IBlockStateWrapper blockState = AIR;
@@ -388,7 +399,7 @@ public class LodDataBuilder
int relSourceBlockX = Math.floorMod(apiChunk.chunkPosX, 4) * LodUtil.CHUNK_WIDTH;
int relSourceBlockZ = Math.floorMod(apiChunk.chunkPosZ, 4) * LodUtil.CHUNK_WIDTH;
FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos);
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos);
for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++)
{
for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++)
core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java
+8 -6
View File
@@ -5,6 +5,7 @@ import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import com.seibel.distanthorizons.core.sql.dto.IBaseDTO;
import com.seibel.distanthorizons.core.util.LodUtil;
@@ -138,9 +139,8 @@ public abstract class AbstractDataSourceHandler
public TDataSource get(long pos)
{
TDataSource dataSource = null;
try
try(TDTO dto = this.repo.getByKey(pos))
{
TDTO dto = this.repo.getByKey(pos);
if (dto != null)
{
try
@@ -152,10 +152,10 @@ public abstract class AbstractDataSourceHandler
{
// Only log each message type once.
// This is done to prevent logging "No compression mode with the value [2]" 10,000 times
// if the user is migrating from a nightly build and used ZStd.
// if the user is migrating from a nightly build and used ZStd.
if (CORRUPT_DATA_ERRORS_LOGGED.add(e.getMessage()))
{
LOGGER.warn("Corrupted data found at pos [" + DhSectionPos.toString(pos) + "]. Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: " + e.getMessage(), e);
LOGGER.warn("Corrupted data found at pos [" + DhSectionPos.toString(pos) + "]. Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + e.getMessage() + "].", e);
}
this.repo.deleteWithKey(pos);
@@ -252,8 +252,10 @@ public abstract class AbstractDataSourceHandler
if (dataModified)
{
// save the updated data to the database
TDTO dto = this.createDtoFromDataSource(recipientDataSource);
this.repo.save(dto);
try (TDTO dto = this.createDtoFromDataSource(recipientDataSource))
{
this.repo.save(dto);
}
for (IDataSourceUpdateFunc<TDataSource> listener : this.dateSourceUpdateListeners)
core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java
-137
View File
@@ -1,137 +0,0 @@
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.core.level.IDhLevel;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
/**
* Data sources are often very large objects and aren't used for very long.
* This means their frequent construction and garbage collection can result in quite a bit of GC pressure.
* By pooling said data sources and reusing them we can drastically reduce this GC pressure and improve
* performance significantly.
*/
public class DataSourcePool<TDataSource extends IDataSource<TDhLevel>, TDhLevel extends IDhLevel>
{
/**
* James tested with a static 25 on a 8 core 16 processor machine and didn't have any issues.
* In most cases the number of pooled sources won't probably even get close to the number of processors,
* but just in case the user has a overkill CPU (or config) this should hopefully prevent thrashing.
*/
private static final int MAX_POOLED_SOURCES = Runtime.getRuntime().availableProcessors() * 2;
private final ArrayList<TDataSource> pooledDataSources = new ArrayList<>();
private final ReentrantLock poolLock = new ReentrantLock();
private final Function<Long, TDataSource> createEmptyDatasourceFunc;
@Nullable
private final IPrepPooledDataSourceFunc<TDataSource, TDhLevel> prepDatasourceFunc;
//=============//
// constructor //
//=============//
public DataSourcePool(Function<Long, TDataSource> createEmptyDatasourceFunc, @Nullable IPrepPooledDataSourceFunc<TDataSource, TDhLevel> prepDatasourceFunc)
{
this.createEmptyDatasourceFunc = createEmptyDatasourceFunc;
this.prepDatasourceFunc = prepDatasourceFunc;
}
//===============//
// pool handlers //
//===============//
/**
* Returns a cleared data source.
* @see DataSourcePool#getPooledSource(long, boolean)
*/
public TDataSource getPooledSource(long pos) { return this.getPooledSource(pos, true);}
/** @return an empty data source if non are cached */
public TDataSource getPooledSource(long pos, boolean clearData)
{
try
{
this.poolLock.lock();
int index = this.pooledDataSources.size() - 1;
if (index == -1)
{
// no pooled sources exist
return this.createEmptyDatasourceFunc.apply(pos);
}
else
{
TDataSource dataSource = this.pooledDataSources.remove(index);
// some data sources may want to handle prep themselves
// (due to needing additional inputs than what this pool keeps track of)
if (this.prepDatasourceFunc != null)
{
this.prepDatasourceFunc.prepDataSource(pos, clearData, dataSource);
}
return dataSource;
}
}
finally
{
this.poolLock.unlock();
}
}
/**
* Doesn't have to be called, if a data source isn't returned, nothing will be leaked.
* It just means a new source must be constructed next time {@link DataSourcePool#getPooledSource} is called.
*/
public void returnPooledDataSource(TDataSource dataSource)
{
if (dataSource == null)
{
return;
}
else if (this.pooledDataSources.size() > MAX_POOLED_SOURCES)
{
return;
}
try
{
this.poolLock.lock();
this.pooledDataSources.add(dataSource);
}
finally
{
this.poolLock.unlock();
}
}
//===============//
// debug methods //
//===============//
/** Returns how many data sources are in the pool */
public int size() { return this.pooledDataSources.size(); }
//================//
// helper classes //
//================//
@FunctionalInterface
public interface IPrepPooledDataSourceFunc<TDataSource extends IDataSource<TDhLevel>, TDhLevel extends IDhLevel>
{
/** @param clearData will be false if the data will be immediately overwritten anyway */
void prepDataSource(long pos, boolean clearData, TDataSource dataSource);
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java
+6 -2
View File
@@ -45,14 +45,18 @@ public class DelayedFullDataSourceSaveCache
// update queue //
//==============//
public void queueDataSourceForUpdateAndSave(FullDataSourceV2 inputDataSource)
/**
* Writing into memory is done synchronously so inputDataSource can
* be closed after this method finishes.
*/
public void writeDataSourceToMemoryAndQueueSave(FullDataSourceV2 inputDataSource)
{
long dataSourcePos = inputDataSource.getPos();
this.dataSourceByPosition.compute(dataSourcePos, (inputPos, temporaryDataSource) ->
{
if (temporaryDataSource == null)
{
temporaryDataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(inputPos);
temporaryDataSource = FullDataSourceV2.createEmpty(inputPos);
}
temporaryDataSource.update(inputDataSource);
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java
+1 -2
View File
@@ -121,9 +121,8 @@ public class FullDataSourceProviderV1<TDhLevel extends IDhLevel>
public FullDataSourceV1 get(Long pos)
{
FullDataSourceV1 dataSource = null;
try
try (FullDataSourceV1DTO dto = this.repo.getByKey(pos))
{
FullDataSourceV1DTO dto = this.repo.getByKey(pos);
if (dto != null)
{
// load from file
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java
+4 -5
View File
@@ -21,7 +21,6 @@ package com.seibel.distanthorizons.core.file.fullDatafile;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.api.internal.ClientApi;
import com.seibel.distanthorizons.core.api.internal.SharedApi;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
@@ -41,7 +40,6 @@ import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.world.EWorldEnvironment;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
@@ -181,10 +179,11 @@ public class FullDataSourceProviderV2
@Override
protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException
{ return dto.createPooledDataSource(this.level.getLevelWrapper()); }
{ return dto.createDataSource(this.level.getLevelWrapper()); }
@Override
protected FullDataSourceV2 makeEmptyDataSource(long pos) { return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true); }
protected FullDataSourceV2 makeEmptyDataSource(long pos)
{ return FullDataSourceV2.createEmpty(pos); }
@@ -286,7 +285,7 @@ public class FullDataSourceProviderV2
}
catch (Exception e)
{
LOGGER.error("issue in update for parent pos: " + parentUpdatePos+ " Error: "+e.getMessage(), e);
LOGGER.error("Unexpected in update for parent pos: [" + DhSectionPos.toString(parentUpdatePos) + "] Error: [" + e.getMessage() + "].", e);
}
finally
{
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java
+54 -42
View File
@@ -29,6 +29,8 @@ import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
@@ -36,6 +38,7 @@ import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
@@ -288,10 +291,10 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
public int getUnsavedDataSourceCount() { return this.delayedFullDataSourceSaveCache.getUnsavedCount(); }
public boolean isFullyGenerated(byte[] columnGenerationSteps)
public boolean isFullyGenerated(ByteArrayList columnGenerationSteps)
{
return IntStream.range(0, columnGenerationSteps.length)
.noneMatch(i -> columnGenerationSteps[i] == EDhApiWorldGenerationStep.EMPTY.value);
return IntStream.range(0, columnGenerationSteps.size())
.noneMatch(i -> columnGenerationSteps.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value);
}
@Override
@@ -307,25 +310,28 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
// don't check any child positions if this position is already fully generated
if (this.repo.existsWithKey(pos))
{
byte[] columnGenerationSteps = this.repo.getColumnGenerationStepForPos(pos);
// shouldn't happen, but just in case
if (columnGenerationSteps != null)
try(PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(1, 0, 0))
{
boolean positionFullyGenerated = true;
// check if any positions are ungenerated
for (int i = 0; i < columnGenerationSteps.length; i++)
ByteArrayList columnGenStepArray = checkout.getByteArray(0);
this.repo.getColumnGenerationStepForPos(pos, columnGenStepArray);
if (!columnGenStepArray.isEmpty())
{
if (columnGenerationSteps[i] == EDhApiWorldGenerationStep.EMPTY.value)
boolean positionFullyGenerated = true;
// check if any positions are ungenerated
for (int i = 0; i < columnGenStepArray.size(); i++)
{
positionFullyGenerated = false;
break;
if (columnGenStepArray.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value)
{
positionFullyGenerated = false;
break;
}
}
if (positionFullyGenerated)
{
return new LongArrayList();
}
}
if (positionFullyGenerated)
{
return new LongArrayList();
}
}
}
@@ -349,36 +355,42 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
}
else
{
byte[] columnGenerationSteps = this.repo.getColumnGenerationStepForPos(genPos);
if (columnGenerationSteps == null)
{
// shouldn't happen, but just in case
return;
}
EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT;
checkWorldGenLoop:
for (int x = 0; x < FullDataSourceV2.WIDTH; x++)
try(PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(1, 0, 0))
{
for (int z = 0; z < FullDataSourceV2.WIDTH; z++)
ByteArrayList columnGenerationSteps = checkout.getByteArray(0);
this.repo.getColumnGenerationStepForPos(genPos, columnGenerationSteps);
if (columnGenerationSteps.isEmpty())
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
byte genStepValue = columnGenerationSteps[index];
if (genStepValue < currentMinWorldGenStep.value)
// shouldn't happen, but just in case
return;
}
checkWorldGenLoop:
for (int x = 0; x < FullDataSourceV2.WIDTH; x++)
{
for (int z = 0; z < FullDataSourceV2.WIDTH; z++)
{
EDhApiWorldGenerationStep newWorldGenStep = EDhApiWorldGenerationStep.fromValue(genStepValue);
if (newWorldGenStep != null && newWorldGenStep.value < currentMinWorldGenStep.value)
int index = FullDataSourceV2.relativePosToIndex(x, z);
byte genStepValue = columnGenerationSteps.getByte(index);
if (genStepValue < currentMinWorldGenStep.value)
{
currentMinWorldGenStep = newWorldGenStep;
EDhApiWorldGenerationStep newWorldGenStep = EDhApiWorldGenerationStep.fromValue(genStepValue);
if (newWorldGenStep != null && newWorldGenStep.value < currentMinWorldGenStep.value)
{
currentMinWorldGenStep = newWorldGenStep;
}
}
if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY)
{
// queue the task
break checkWorldGenLoop;
}
}
if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY)
{
// queue the task
break checkWorldGenLoop;
}
}
}
@@ -447,7 +459,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
{
return (dataSource) ->
{
GeneratedFullDataSourceProvider.this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(dataSource);
GeneratedFullDataSourceProvider.this.delayedFullDataSourceSaveCache.writeDataSourceToMemoryAndQueueSave(dataSource);
};
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java
+5 -1
View File
@@ -25,7 +25,9 @@ import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.generation.RemoteWorldRetrievalQueue;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.level.WorldGenModule;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.multiplayer.client.SyncOnLoadRequestQueue;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.File;
@@ -39,6 +41,8 @@ import java.util.concurrent.TimeUnit;
*/
public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvider
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@Nullable
private final SyncOnLoadRequestQueue syncOnLoadRequestQueue;
private final Set<Long> visitedPositions = Collections.newSetFromMap(CacheBuilder.newBuilder()
@@ -95,7 +99,7 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide
Long timestamp = this.getTimestampForPos(pos);
if (timestamp != null)
{
this.syncOnLoadRequestQueue.submitRequest(pos, timestamp, this.delayedFullDataSourceSaveCache::queueDataSourceForUpdateAndSave);
this.syncOnLoadRequestQueue.submitRequest(pos, timestamp, this.delayedFullDataSourceSaveCache::writeDataSourceToMemoryAndQueueSave);
}
return super.get(pos);
core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java
+11 -4
View File
@@ -392,15 +392,22 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
try
{
IChunkWrapper chunk = WRAPPER_FACTORY.createChunkWrapper(generatedObjectArray);
FullDataSourceV2 dataSource = LodDataBuilder.createFromChunk(chunk);
LodUtil.assertTrue(dataSource != null);
dataSourceConsumer.accept(dataSource);
try (FullDataSourceV2 dataSource = LodDataBuilder.createFromChunk(chunk))
{
LodUtil.assertTrue(dataSource != null);
dataSourceConsumer.accept(dataSource);
}
}
catch (ClassCastException e)
{
LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e);
Config.Common.WorldGenerator.enableDistantGeneration.set(false);
}
catch (Exception e)
{
LOGGER.error("Unexpected world generator error. Error: [" + e.getMessage() + "]. World generator disabled.", e);
Config.Common.WorldGenerator.enableDistantGeneration.set(false);
}
}
);
}
@@ -435,7 +442,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
case API_DATA_SOURCES:
{
// done to reduce GC overhead
FullDataSourceV2 pooledDataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(requestPos);
FullDataSourceV2 pooledDataSource = FullDataSourceV2.createEmpty(requestPos);
// set here so the API user doesn't have to pass in this value anywhere themselves
pooledDataSource.setRunApiChunkValidation(this.generator.runApiValidation());
core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java
+21 -19
View File
@@ -27,7 +27,6 @@ import com.seibel.distanthorizons.core.generation.DhLightingEngine;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler;
import com.seibel.distanthorizons.core.render.renderer.generic.CloudRenderHandler;
import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO;
@@ -147,27 +146,30 @@ public abstract class AbstractDhLevel implements IDhLevel
@Override
public void updateChunkAsync(IChunkWrapper chunkWrapper, int chunkHash)
{
FullDataSourceV2 dataSource = FullDataSourceV2.createFromChunk(chunkWrapper);
if (dataSource == null)
// data source synchronously written to memory so it can be safely closed
try (FullDataSourceV2 dataSource = FullDataSourceV2.createFromChunk(chunkWrapper))
{
// This can happen if, among other reasons, a chunk save is superseded by a later event
return;
}
this.updatedChunkPosSetBySectionPos.compute(dataSource.getPos(), (dataSourcePos, chunkPosSet) ->
{
if (chunkPosSet == null)
if (dataSource == null)
{
chunkPosSet = new HashSet<>();
// This can happen if, among other reasons, a chunk save is superseded by a later event
return;
}
chunkPosSet.add(chunkWrapper.getChunkPos());
return chunkPosSet;
});
this.updatedChunkHashesByChunkPos.put(chunkWrapper.getChunkPos(), chunkHash);
// batch updates to reduce overhead when flying around or breaking/placing a lot of blocks in an area
this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(dataSource);
this.updatedChunkPosSetBySectionPos.compute(dataSource.getPos(), (dataSourcePos, chunkPosSet) ->
{
if (chunkPosSet == null)
{
chunkPosSet = new HashSet<>();
}
chunkPosSet.add(chunkWrapper.getChunkPos());
return chunkPosSet;
});
this.updatedChunkHashesByChunkPos.put(chunkWrapper.getChunkPos(), chunkHash);
// batch updates to reduce overhead when flying around or breaking/placing a lot of blocks in an area
this.delayedFullDataSourceSaveCache.writeDataSourceToMemoryAndQueueSave(dataSource);
}
}
private void onDataSourceSave(FullDataSourceV2 fullDataSource)
core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java
+1 -1
View File
@@ -145,7 +145,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
}
this.beaconBeamDataHandler.setBeaconBeamsForPos(dataSourceDto.pos, message.payload.beaconBeams);
this.updateDataSourcesAsync(dataSourceDto.createPooledDataSource(this.levelWrapper));
this.updateDataSourcesAsync(dataSourceDto.createDataSource(this.levelWrapper));
}
catch (Exception e)
{
core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java
+4 -1
View File
@@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.api.internal.SharedApi;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.jar.ModJarInfo;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.render.RenderBufferHandler;
import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
@@ -30,7 +31,6 @@ import com.seibel.distanthorizons.core.util.threading.RateLimitedThreadPoolExecu
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.world.AbstractDhWorld;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import com.seibel.distanthorizons.coreapi.DependencyInjection.DependencyInjector;
import com.seibel.distanthorizons.coreapi.ModInfo;
import com.seibel.distanthorizons.coreapi.util.StringUtil;
import org.apache.logging.log4j.LogManager;
@@ -107,6 +107,9 @@ public class F3Screen
messageList.add(getThreadPoolStatString("LOD Builder", lodBuilderPool));
messageList.add(getThreadPoolStatString("Buffer Builder", bufferBuilderPool));
messageList.add("");
// object pools
PhantomArrayListPool.INSTANCE.addDebugMenuStringsToList(messageList);
messageList.add("");
// chunk updates
messageList.add(SharedApi.INSTANCE.getDebugMenuString());
messageList.add("");
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java
+5 -6
View File
@@ -20,7 +20,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.ratelimiting.SupplierBasedRateLimiter;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.world.DhApiWorldProxy;
@@ -30,7 +29,6 @@ import org.apache.logging.log4j.LogManager;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import java.awt.*;
import java.io.IOException;
import java.util.*;
import java.util.List;
import java.util.concurrent.*;
@@ -235,11 +233,12 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
try
{
this.level.getBeaconBeamDataHandler().setBeaconBeamsForPos(dataSourceDto.pos, response.payload.beaconBeams);
FullDataSourceV2 fullDataSource = dataSourceDto.createPooledDataSource(this.level.getLevelWrapper());
entry.dataSourceConsumer.accept(fullDataSource);
FullDataSourceV2.DATA_SOURCE_POOL.returnPooledDataSource(fullDataSource);
try (FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper()))
{
entry.dataSourceConsumer.accept(fullDataSource);
}
}
catch (IOException | DataCorruptedException | InterruptedException e)
catch (Exception e)
{
throw new RuntimeException(e);
}
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayload.java
+5 -4
View File
@@ -45,10 +45,11 @@ public class FullDataPayload implements INetworkObject, AutoCloseable
try
{
EDhApiDataCompressionMode compressionMode = Config.Common.LodBuilding.dataCompression.get();
FullDataSourceV2DTO dataSourceDto = FullDataSourceV2DTO.CreateFromDataSource(fullDataSource, compressionMode);
this.dtoBuffer = ByteBufAllocator.DEFAULT.buffer();
dataSourceDto.encode(this.dtoBuffer);
try (FullDataSourceV2DTO dataSourceDto = FullDataSourceV2DTO.CreateFromDataSource(fullDataSource, compressionMode))
{
this.dtoBuffer = ByteBufAllocator.DEFAULT.buffer();
dataSourceDto.encode(this.dtoBuffer);
}
}
catch (IOException e)
{
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/fullData/FullDataPayloadReceiver.java
+1 -1
View File
@@ -76,7 +76,7 @@ public class FullDataPayloadReceiver implements AutoCloseable
try
{
return INetworkObject.decodeToInstance(FullDataSourceV2DTO.CreateEmptyDataSource(), compositeByteBuffer);
return INetworkObject.decodeToInstance(FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding(), compositeByteBuffer);
}
finally
{
core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java
+103
View File
@@ -0,0 +1,103 @@
package com.seibel.distanthorizons.core.pooling;
import com.seibel.distanthorizons.core.util.ListUtil;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import java.util.ArrayList;
/**
* This keeps track of all the poolable
* arrays that can be retrieved via the {@link PhantomArrayListPool}.
*
* @see PhantomArrayListParent
* @see PhantomArrayListPool
*/
public class PhantomArrayListCheckout implements AutoCloseable
{
private final ArrayList<ByteArrayList> byteArrayLists = new ArrayList<>();
private final ArrayList<ShortArrayList> shortArrayLists = new ArrayList<>();
private final ArrayList<LongArrayList> longArrayLists = new ArrayList<>();
//=========//
// setters //
//=========//
public void addByteArrayList(ByteArrayList list) { this.byteArrayLists.add(list); }
public void addShortArrayList(ShortArrayList list) { this.shortArrayLists.add(list); }
public void addLongArrayList(LongArrayList list) { this.longArrayLists.add(list); }
//=========//
// getters //
//=========//
public int getByteArrayCount() { return this.byteArrayLists.size(); }
public int getShortArrayCount() { return this.shortArrayLists.size(); }
public int getLongArrayCount() { return this.longArrayLists.size(); }
public ByteArrayList getByteArray(int index) { return this.getByteArray(index, 0); }
public ByteArrayList getByteArray(int index, int size)
{
ByteArrayList list = this.byteArrayLists.get(index);
if (size != 0)
{
ListUtil.clearAndSetSize(list, size);
}
else
{
list.clear();
}
return list;
}
public ShortArrayList getShortArray(int index) { return this.getShortArray(index, 0); }
public ShortArrayList getShortArray(int index, int size)
{
ShortArrayList list = this.shortArrayLists.get(index);
if (size != 0)
{
ListUtil.clearAndSetSize(list, size);
}
else
{
list.clear();
}
return list;
}
public LongArrayList getLongArray(int index) { return this.getLongArray(index, 0); }
public LongArrayList getLongArray(int index, int size)
{
LongArrayList list = this.longArrayLists.get(index);
if (size != 0)
{
ListUtil.clearAndSetSize(list, size);
}
else
{
list.clear();
}
return list;
}
public ArrayList<ByteArrayList> getAllByteArrays() { return this.byteArrayLists; }
public ArrayList<ShortArrayList> getAllShortArrays() { return this.shortArrayLists; }
public ArrayList<LongArrayList> getAllLongArrays() { return this.longArrayLists; }
@Override
public void close()
{
PhantomArrayListPool.INSTANCE.returnCheckout(this);
}
}
core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java
+148
View File
@@ -0,0 +1,148 @@
package com.seibel.distanthorizons.core.pooling;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.lang.ref.PhantomReference;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
/**
* Any object that needs pooled arrays should extend this object.
* This handles setting up and tracking the necessary {@link PhantomReference}'s
* needed to make sure none of the arrays are leaked.
* However, if possible, the implementing object should be closed
* instead via a try-resource block as that will reduce the number of
* unnecessary arrays created.
*
* @see PhantomArrayListCheckout
* @see PhantomArrayListPool
*/
public abstract class PhantomArrayListParent implements AutoCloseable
{
private static final Logger LOGGER = LogManager.getLogger();
private static final ConcurrentHashMap<Reference<? extends PhantomArrayListParent>, PhantomArrayListCheckout>
PHANTOM_REF_TO_CHECKOUT = new ConcurrentHashMap<>();
private static final ReferenceQueue<PhantomArrayListParent> PHANTOM_REF_QUEUE = new ReferenceQueue<>();
private static final int PHANTOM_REF_CHECK_TIME_IN_MS = 5 * 1000;
private static final ThreadPoolExecutor RECYCLER_THREAD = ThreadUtil.makeSingleThreadPool("Phantom Array Recycler");
private final PhantomReference<PhantomArrayListParent> phantomReference;
/**
* It's recommended to set this as null after the child's constructor
* finishes to show the pooled arrays have all been accessed
*/
protected PhantomArrayListCheckout pooledArraysCheckout;
//====================//
// static constructor //
//====================//
static { RECYCLER_THREAD.execute(() -> runPhantomReferenceCleanupLoop()); }
private static void runPhantomReferenceCleanupLoop()
{
while (true)
{
try
{
try
{
Thread.sleep(PHANTOM_REF_CHECK_TIME_IN_MS);
}
catch (InterruptedException ignore) { }
int returnedByteArrayCount = 0;
int returnedShortArrayCount = 0;
int returnedLongArrayCount = 0;
Reference<? extends PhantomArrayListParent> phantomRef = PHANTOM_REF_QUEUE.poll();
while (phantomRef != null)
{
// return the pooled arrays
PhantomArrayListCheckout checkout = PHANTOM_REF_TO_CHECKOUT.remove(phantomRef);
if (checkout != null)
{
returnedByteArrayCount += checkout.getByteArrayCount();
returnedShortArrayCount += checkout.getShortArrayCount();
returnedLongArrayCount += checkout.getLongArrayCount();
PhantomArrayListPool.INSTANCE.returnCheckout(checkout);
}
else
{
// shouldn't happen, but just in case
LOGGER.warn("Unable to find checkout for phantom reference ["+phantomRef+"], arrays will need to be recreated.");
}
phantomRef = PHANTOM_REF_QUEUE.poll();
}
if (returnedByteArrayCount != 0 && returnedLongArrayCount != 0)
{
// we only want to log when arrays have been returned
//LOGGER.info("Returned byte:["+F3Screen.NUMBER_FORMAT.format(returnedByteArrayCount)+"], short:["+F3Screen.NUMBER_FORMAT.format(returnedShortArrayCount)+"], long:["+F3Screen.NUMBER_FORMAT.format(returnedLongArrayCount)+"].");
// since this is just for debugging it only needs to be recalculated once in a while
PhantomArrayListPool.INSTANCE.recalculateSizeForDebugging();
}
}
catch (Exception e)
{
LOGGER.error("Unexpected error in phantom pool return thread, error: [" + e.getMessage() + "].", e);
}
}
}
//=============//
// constructor //
//=============//
/** The Array counts can be 0 or greater. */
public PhantomArrayListParent(int byteArrayCount, int shortArrayCount, int longArrayCount)
{
if (byteArrayCount < 0 || shortArrayCount < 0 || longArrayCount < 0)
{
throw new IllegalArgumentException("Can't get a negative number of pooled arrays.");
}
this.phantomReference = new PhantomReference<>(this, PHANTOM_REF_QUEUE);
this.pooledArraysCheckout = PhantomArrayListPool.INSTANCE.checkoutArrays(byteArrayCount, shortArrayCount, longArrayCount);
PHANTOM_REF_TO_CHECKOUT.put(this.phantomReference, this.pooledArraysCheckout);
}
//================//
// base overrides //
//================//
@Override
public void close() //throws Exception
{
try
{
this.phantomReference.clear();
PhantomArrayListCheckout checkout = PHANTOM_REF_TO_CHECKOUT.remove(this.phantomReference);
PhantomArrayListPool.INSTANCE.returnCheckout(checkout);
}
catch (Exception e)
{
LOGGER.error("", e);
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java
+269
View File
@@ -0,0 +1,269 @@
package com.seibel.distanthorizons.core.pooling;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.coreapi.util.StringUtil;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
/**
* DH uses a lot of potentially large arrays of {@link Byte}s and {@link Long}s.
* In order to reduce Garbage Collector (GC) stuttering and array allocation overhead
* we pool these arrays when possible. <br><br>
*
* How pooled arrays can be returned: <br>
* 1. <b> Closing the {@link PhantomArrayListParent} </b> <br>
* The fastest and most efficient method of returning pooled arrays
* is to call {@link AutoCloseable#close()}. <br><br>
*
* 2. <b> {@link PhantomArrayListParent} Garbage Collection </b> <br>
* Some objects are used across many different threads and
* cleanly closing them is impossible, so when the {@link PhantomArrayListParent}
* is automatically garbage collected we recover and recycle any
* arrays it checked out.
* This is less efficient since it may allow a lot of additional arrays to
* be created while we wait for the garbage collector to run, but
* does prevent any leaks from {@link PhantomArrayListParent} that weren't closed.
*/
public class PhantomArrayListPool
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public static final PhantomArrayListPool INSTANCE = new PhantomArrayListPool();
/** needed since our pools aren't thread safe */
private final ReentrantLock poolLock = new ReentrantLock();
private final ArrayList<ByteArrayList> pooledByteArrays = new ArrayList<>();
private final ArrayList<ShortArrayList> pooledShortArrays = new ArrayList<>();
private final ArrayList<LongArrayList> pooledLongArrays = new ArrayList<>();
/** counts how many byte arrays have been created by this pool */
private final AtomicInteger totalByteArrayCountRef = new AtomicInteger(0);
/** counts how many short arrays have been created by this pool */
private final AtomicInteger totalShortArrayCountRef = new AtomicInteger(0);
/** counts how many long arrays have been created by this pool */
private final AtomicInteger totalLongArrayCountRef = new AtomicInteger(0);
/** used for debugging, represents an estimate for how many bytes the byte[] pool contains */
private long lastBytePoolSizeInBytes = -1;
/** used for debugging, represents an estimate for how many bytes the short[] pool contains */
private long lastShortPoolSizeInBytes = -1;
/** used for debugging, represents an estimate for how many bytes the long[] pool contains */
private long lastLongPoolSizeInBytes = -1;
//=============//
// constructor //
//=============//
private PhantomArrayListPool()
{
// create initial arrays
PhantomArrayListCheckout checkout = this.checkoutArrays(2_000, 100, 100_000);
this.returnCheckout(checkout);
}
//==============//
// get checkout //
//==============//
public PhantomArrayListCheckout checkoutArrays(int byteArrayCount, int shortArrayCount, int longArrayCount)
{
try
{
this.poolLock.lock();
PhantomArrayListCheckout checkout = new PhantomArrayListCheckout();
// byte
for (int i = 0; i < byteArrayCount; i++)
{
checkout.addByteArrayList(getPooledArray(this.pooledByteArrays, this::createEmptyByteArrayList));
}
// short
for (int i = 0; i < shortArrayCount; i++)
{
checkout.addShortArrayList(getPooledArray(this.pooledShortArrays, this::createEmptyShortArrayList));
}
// long
for (int i = 0; i < longArrayCount; i++)
{
checkout.addLongArrayList(getPooledArray(this.pooledLongArrays, this::createEmptyLongArrayList));
}
return checkout;
}
finally
{
this.poolLock.unlock();
}
}
// array constructors //
private ByteArrayList createEmptyByteArrayList()
{
//LOGGER.error("created new byte array");
this.totalByteArrayCountRef.getAndIncrement();
return new ByteArrayList();
}
private ShortArrayList createEmptyShortArrayList()
{
//LOGGER.error("created new short array");
this.totalShortArrayCountRef.getAndIncrement();
return new ShortArrayList();
}
private LongArrayList createEmptyLongArrayList()
{
//LOGGER.error("created new long array");
this.totalLongArrayCountRef.getAndIncrement();
return new LongArrayList();
}
// internal pool handler //
private static <T extends List<?>> T getPooledArray(ArrayList<T> pool, Supplier<T> emptyArrayCreatorFunc)
{
int index = pool.size() - 1;
if (index == -1)
{
// no pooled sources exist
return emptyArrayCreatorFunc.get();
}
else
{
T array = pool.remove(index);
array.clear();
return array;
}
}
//=================//
// return checkout //
//=================//
public void returnCheckout(PhantomArrayListCheckout checkout)
{
if (checkout == null)
{
throw new IllegalArgumentException("Null phantom checkout, memory leak in progress...");
}
this.poolLock.lock();
try
{
// In James' testing pooling the checkout object wasn't necessary
// since it is relatively small and short lived it appears
// the GC can handle quickly discarding it.
this.pooledByteArrays.addAll(checkout.getAllByteArrays());
this.pooledShortArrays.addAll(checkout.getAllShortArrays());
this.pooledLongArrays.addAll(checkout.getAllLongArrays());
//LOGGER.info("Returned ["+checkout.byteArrayLists.size()+"/"+this.pooledByteArrays.size()+"] bytes and ["+checkout.longArrayLists.size()+"/"+this.pooledLongArrays.size()+"] longs.");\
}
finally
{
this.poolLock.unlock();
}
}
//===============//
// debug methods //
//===============//
public void addDebugMenuStringsToList(List<String> messageList)
{
// total (all time created) count
String byteArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalByteArrayCountRef.get());
String shortArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalShortArrayCountRef.get());
String longArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalLongArrayCountRef.get());
// inactive items in pool
String bytePoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledByteArrays.size());
String shortPoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledShortArrays.size());
String longPoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledLongArrays.size());
// pool byte size
String bytePoolSizeInBytes = (this.lastBytePoolSizeInBytes != -1)
? " ~" + StringUtil.convertBytesToHumanReadable(this.lastBytePoolSizeInBytes)
: "";
String shortPoolSizeInBytes = (this.lastShortPoolSizeInBytes != -1)
? " ~" + StringUtil.convertBytesToHumanReadable(this.lastShortPoolSizeInBytes)
: "";
String longPoolSizeInBytes = (this.lastLongPoolSizeInBytes != -1)
? " ~" + StringUtil.convertBytesToHumanReadable(this.lastLongPoolSizeInBytes)
: "";
messageList.add("Pools:");
messageList.add("byte[]: "+bytePoolCount+"/"+byteArrayTotalCount + bytePoolSizeInBytes);
messageList.add("short[]: "+shortPoolCount+"/"+shortArrayTotalCount + shortPoolSizeInBytes);
messageList.add("long[]: "+longPoolCount+"/"+longArrayTotalCount + longPoolSizeInBytes);
}
/**
* shouldn't be called on the render thread as it can
* take 10's of milliseconds to complete.
*/
public void recalculateSizeForDebugging()
{
this.poolLock.lock();
try
{
// byte
long bytePoolByteSize = estimateMemoryUsage(this.pooledByteArrays, Byte.BYTES);
this.lastBytePoolSizeInBytes = Math.max(bytePoolByteSize, this.lastBytePoolSizeInBytes);
// short
long shortPoolByteSize = estimateMemoryUsage(this.pooledShortArrays, Short.BYTES);
this.lastShortPoolSizeInBytes = Math.max(shortPoolByteSize, this.lastShortPoolSizeInBytes);
// long
long longPoolByteSize = estimateMemoryUsage(this.pooledLongArrays, Long.BYTES);
this.lastLongPoolSizeInBytes = Math.max(longPoolByteSize, this.lastLongPoolSizeInBytes);
}
finally
{
this.poolLock.unlock();
}
}
private static <T extends Collection<?>> long estimateMemoryUsage(ArrayList<T> pool, long elementSizeInBytes)
{
long longByteSize = 0;
for (int i = 0; i < pool.size(); i++)
{
// Object overhead + capacity of underlying array * size of Long (8 bytes)
long overhead = Byte.SIZE * 4;
long arraySize = (long)pool.get(i).size() * elementSizeInBytes;
longByteSize += overhead + arraySize;
}
return longByteSize;
}
}
core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java
+1
View File
@@ -434,6 +434,7 @@ public class DhSectionPos
// Base methods //
//==============//
/** Example: "6*1,-3" */
public static String toString(long pos) { return getDetailLevel(pos) + "*" + getX(pos) + "," + getZ(pos); }
public static int hashCode(long pos) { return Long.hashCode(pos); }
core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java
+2 -3
View File
@@ -27,7 +27,6 @@ import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodQuad
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.file.DataSourcePool;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -632,7 +631,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
/**
* Used to keep track of whether a {@link ColumnRenderSource} {@link CompletableFuture}
* is in use or not, and if not in use returns the data source to the {@link DataSourcePool}. <br> <br>
* is in use or not. <br> <br>
*
* This reduces GC overhead by pooling shared {@link ColumnRenderSource}.
*/
@@ -687,7 +686,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
ColumnRenderSource source = this.future.getNow(null);
if (source != null)
{
ColumnRenderSource.DATA_SOURCE_POOL.returnPooledDataSource(source);
source.close();
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java
+14 -10
View File
@@ -26,9 +26,7 @@ import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.*;
import java.util.ArrayList;
import java.util.Map;
import java.util.Scanner;
@@ -130,17 +128,23 @@ public class DatabaseUpdater
}
catch (SQLException e)
{
connection.rollback();
LOGGER.error(
"Unexpected SQL Error: ["+e.getMessage()+"] " +
"returned for auto update script: [" + resource.name + "], " +
"query: [" + fileUpdateSqlArray[sqlIndex] + "]. " +
"Changes should have been rolled back.", new SQLException());
LOGGER.error("Unexpected SQL Error: ["+e.getMessage()+"] \n" +
"returned for auto update script: [" + resource.name + "], \n" +
"query: [" + fileUpdateSqlArray[sqlIndex] + "]. \n" +
"Changes should have been rolled back.",
new SQLException()
);
if (transactScript)
{
connection.rollback();
}
throw e;
}
if (transactScript)
{
{
// revert to default setting
connection.setAutoCommit(true);
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/BeaconBeamDTO.java
+14 -9
View File
@@ -46,15 +46,6 @@ public class BeaconBeamDTO implements IBaseDTO<DhBlockPos>, INetworkObject
//===========//
// overrides //
//===========//
@Override
public DhBlockPos getKey() { return this.blockPos; }
//=========//
// network //
//=========//
@@ -73,4 +64,18 @@ public class BeaconBeamDTO implements IBaseDTO<DhBlockPos>, INetworkObject
this.color = new Color(in.readInt());
}
//===========//
// overrides //
//===========//
@Override
public DhBlockPos getKey() { return this.blockPos; }
@Override
public void close()
{ /* no closing needed */ }
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/ChunkHashDTO.java
+7
View File
@@ -26,6 +26,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
@@ -69,4 +70,10 @@ public class ChunkHashDTO implements IBaseDTO<DhChunkPos>
@Override
public DhChunkPos getKey() { return this.pos; }
@Override
public void close()
{ /* no closing needed */ }
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java
+3
View File
@@ -85,5 +85,8 @@ public class FullDataSourceV1DTO implements IBaseDTO<Long>
@Override
public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
@Override
public void close()
{ /* no closing needed */ }
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+134 -136
View File
@@ -23,27 +23,29 @@ import com.google.common.base.MoreObjects;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.api.internal.ClientApi;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.network.INetworkObject;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.ListUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import io.netty.buffer.ByteBuf;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
/** handles storing {@link FullDataSourceV2}'s in the database. */
public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
public class FullDataSourceV2DTO
extends PhantomArrayListParent
implements IBaseDTO<Long>, INetworkObject, AutoCloseable
{
public static final boolean VALIDATE_INPUT_DATAPOINTS = true;
@@ -55,18 +57,14 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
/** only for the data array */
public int dataChecksum;
// TODO pool byte arrays, and use JDBC blobs or stream said arrays, to potentially reduce GC load
//resultSet.getBinaryStream(0);
//resultSet.getBlob(0).free();
public byte[] compressedDataByteArray;
public ByteArrayList compressedDataByteArray;
/** @see EDhApiWorldGenerationStep */
public byte[] compressedColumnGenStepByteArray;
public ByteArrayList compressedColumnGenStepByteArray;
/** @see EDhApiWorldCompressionMode */
public byte[] compressedWorldCompressionModeByteArray;
public ByteArrayList compressedWorldCompressionModeByteArray;
public byte[] compressedMappingByteArray;
public ByteArrayList compressedMappingByteArray;
public byte dataFormatVersion;
public byte compressionModeValue;
@@ -78,56 +76,47 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
//=============//
// constructor //
//=============//
//==============//
// constructors //
//==============//
public static FullDataSourceV2DTO CreateFromDataSource(FullDataSourceV2 dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
byte[] dataPointByteArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum);
byte[] compressedWorldGenStepByteArray = writeGenerationStepsToBlob(dataSource.columnGenerationSteps, compressionModeEnum);
byte[] compressedWorldCompressionModeByteArray = writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, compressionModeEnum);
byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum);
FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding();
int checksum = (dataSource.mapping.hashCode() * 4217) + dataSource.hashCode();
// populate arrays
writeDataSourceDataArrayToBlob(dataSource.dataPoints, dto.compressedDataByteArray, compressionModeEnum);
writeGenerationStepsToBlob(dataSource.columnGenerationSteps, dto.compressedColumnGenStepByteArray, compressionModeEnum);
writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, dto.compressedWorldCompressionModeByteArray, compressionModeEnum);
writeDataMappingToBlob(dataSource.mapping, dto.compressedMappingByteArray, compressionModeEnum);
return new FullDataSourceV2DTO(
dataSource.getPos(),
checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum.value, dataPointByteArray,
dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime,
mappingByteArray, dataSource.applyToParent,
dataSource.levelMinY
);
// populate individual variables
{
dto.pos = dataSource.getPos();
dto.dataChecksum = (dataSource.mapping.hashCode() * 4217) + dataSource.hashCode();
dto.dataFormatVersion = FullDataSourceV2.DATA_FORMAT_VERSION;
dto.compressionModeValue = compressionModeEnum.value;
dto.lastModifiedUnixDateTime = dataSource.lastModifiedUnixDateTime;
dto.createdUnixDateTime = dataSource.createdUnixDateTime;
dto.applyToParent = dataSource.applyToParent;
dto.levelMinY = dataSource.levelMinY;
}
return dto;
}
/** Should only be used for subsequent decoding */
public static FullDataSourceV2DTO CreateEmptyDataSource() { return new FullDataSourceV2DTO(); }
private FullDataSourceV2DTO() { }
public FullDataSourceV2DTO(
long pos,
int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, byte compressionModeValue, byte[] compressedDataByteArray,
long lastModifiedUnixDateTime, long createdUnixDateTime,
byte[] compressedMappingByteArray, boolean applyToParent,
int levelMinY)
public static FullDataSourceV2DTO CreateEmptyDataSourceForDecoding() { return new FullDataSourceV2DTO(); }
private FullDataSourceV2DTO()
{
this.pos = pos;
this.dataChecksum = dataChecksum;
this.compressedColumnGenStepByteArray = compressedColumnGenStepByteArray;
this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray;
super(4, 0, 0);
this.dataFormatVersion = dataFormatVersion;
this.compressionModeValue = compressionModeValue;
this.compressedDataByteArray = this.pooledArraysCheckout.getByteArray(0);
this.compressedColumnGenStepByteArray = this.pooledArraysCheckout.getByteArray(1);
this.compressedWorldCompressionModeByteArray = this.pooledArraysCheckout.getByteArray(2);
this.compressedMappingByteArray = this.pooledArraysCheckout.getByteArray(3);
this.compressedDataByteArray = compressedDataByteArray;
this.compressedMappingByteArray = compressedMappingByteArray;
this.applyToParent = applyToParent;
this.lastModifiedUnixDateTime = lastModifiedUnixDateTime;
this.createdUnixDateTime = createdUnixDateTime;
this.levelMinY = levelMinY;
this.pooledArraysCheckout = null;
}
@@ -136,15 +125,13 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
// data source population //
//========================//
public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
{
FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(this.pos, false);
return this.populateDataSource(dataSource, levelWrapper);
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos);
this.internalPopulateDataSource(dataSource, levelWrapper, false);
return dataSource;
}
public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
{ return this.internalPopulateDataSource(dataSource, levelWrapper, false); }
/**
* May be missing one or more data fields. <br>
* Designed to be used without access to Minecraft or any supporting objects.
@@ -173,9 +160,9 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
}
dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, compressionModeEnum);
dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, compressionModeEnum);
dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, compressionModeEnum);
readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum);
readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum);
readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum);
dataSource.mapping.clear(dataSource.getPos());
// should only be null when used in a unit test
@@ -211,7 +198,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
// (de)serializing //
//=================//
private static byte[] writeDataSourceDataArrayToBlob(LongArrayList[] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
private static void writeDataSourceDataArrayToBlob(LongArrayList[] inputDataArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
// write the outputs to a stream to prep for writing to the database
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -225,7 +212,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH;
for (int xz = 0; xz < dataArrayLength; xz++)
{
LongArrayList dataColumn = dataArray[xz];
LongArrayList dataColumn = inputDataArray[xz];
// write column length
short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0;
@@ -244,19 +231,17 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
// generate the checksum
compressedOut.flush();
byteArrayOutputStream.close();
return byteArrayOutputStream.toByteArray();
outputByteArray.addElements(0, byteArrayOutputStream.toByteArray());
}
private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
private static void readBlobToDataSourceDataArray(ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray);
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.toByteArray());
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
// read the data
int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH;
LongArrayList[] dataArray = new LongArrayList[dataArrayLength];
for (int xz = 0; xz < dataArray.length; xz++)
for (int xz = 0; xz < dataArrayLength; xz++)
{
// read the column length
short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later
@@ -265,7 +250,8 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
throw new DataCorruptedException("Read DataSource Blob data at index ["+xz+"], column length ["+dataColumnLength+"] should be greater than zero.");
}
LongArrayList dataColumn = new LongArrayList(new long[dataColumnLength]);
LongArrayList dataColumn = outputDataLongArray[xz];
ListUtil.clearAndSetSize(dataColumn, dataColumnLength);
// read column data (will be skipped if no data was present)
for (int y = 0; y < dataColumnLength; y++)
@@ -277,38 +263,32 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
}
dataColumn.set(y, dataPoint);
}
dataArray[xz] = dataColumn;
}
return dataArray;
}
private static byte[] writeGenerationStepsToBlob(byte[] columnGenStepByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
private static void writeGenerationStepsToBlob(ByteArrayList inputColumnGenStepByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum);
compressedOut.write(columnGenStepByteArray);
for (int i = 0; i < inputColumnGenStepByteArray.size(); i++)
{
compressedOut.writeByte(inputColumnGenStepByteArray.getByte(i));
}
compressedOut.flush();
byteArrayOutputStream.close();
return byteArrayOutputStream.toByteArray();
outputByteArray.addElements(0, byteArrayOutputStream.toByteArray());
}
private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
private static void readBlobToGenerationSteps(ByteArrayList inputCompressedDataByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray);
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.toByteArray());
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
try
{
byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
compressedIn.readFully(columnGenStepByteArray);
return columnGenStepByteArray;
compressedIn.readFully(outputByteArray.elements(), 0, FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH);
}
catch (EOFException e)
{
@@ -317,31 +297,37 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
}
private static byte[] writeWorldCompressionModeToBlob(byte[] worldCompressionModeByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
private static void writeWorldCompressionModeToBlob(ByteArrayList inputWorldCompressionModeByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum);
compressedOut.write(worldCompressionModeByteArray);
for (int i = 0; i < inputWorldCompressionModeByteArray.size(); i++)
{
compressedOut.write(inputWorldCompressionModeByteArray.getByte(i));
}
compressedOut.flush();
byteArrayOutputStream.close();
return byteArrayOutputStream.toByteArray();
outputByteArray.addElements(0, byteArrayOutputStream.toByteArray());
}
private static byte[] readBlobToWorldCompressionMode(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException
private static void readBlobToWorldCompressionMode(ByteArrayList inputCompressedDataByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray);
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.toByteArray());
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
byte[] worldCompressionModeByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
compressedIn.readFully(worldCompressionModeByteArray);
return worldCompressionModeByteArray;
try
{
compressedIn.readFully(outputByteArray.elements(), 0, FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH);
}
catch (EOFException e)
{
throw new DataCorruptedException(e);
}
}
private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException
private static void writeDataMappingToBlob(FullDataPointIdMap mapping, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum);
@@ -350,12 +336,11 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
compressedOut.flush();
byteArrayOutputStream.close();
return byteArrayOutputStream.toByteArray();
outputByteArray.addElements(0, byteArrayOutputStream.toByteArray());
}
private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, long pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException
private static FullDataPointIdMap readBlobToDataMapping(ByteArrayList compressedMappingByteArray, long pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray);
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray.toByteArray());
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper);
@@ -371,27 +356,36 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
@Override
public void encode(ByteBuf out)
{
out.writeLong(this.pos);
out.writeInt(this.dataChecksum);
out.writeInt(this.compressedDataByteArray.length);
out.writeBytes(this.compressedDataByteArray);
out.writeInt(this.compressedColumnGenStepByteArray.length);
out.writeBytes(this.compressedColumnGenStepByteArray);
out.writeInt(this.compressedWorldCompressionModeByteArray.length);
out.writeBytes(this.compressedWorldCompressionModeByteArray);
out.writeInt(this.compressedMappingByteArray.length);
out.writeBytes(this.compressedMappingByteArray);
out.writeByte(this.dataFormatVersion);
out.writeByte(this.compressionModeValue);
out.writeBoolean(this.applyToParent);
out.writeLong(this.lastModifiedUnixDateTime);
out.writeLong(this.createdUnixDateTime);
try
{
out.writeLong(this.pos);
out.writeInt(this.dataChecksum);
out.writeInt(this.compressedDataByteArray.size());
// writing a stream can throw an IO Exception, however since this is just a array wrapper
// nothing should be thrown unless we have the size wrong
out.writeBytes(new ByteArrayInputStream(this.compressedDataByteArray.elements()), this.compressedDataByteArray.size());
out.writeInt(this.compressedColumnGenStepByteArray.size());
out.writeBytes(new ByteArrayInputStream(this.compressedColumnGenStepByteArray.elements()), this.compressedColumnGenStepByteArray.size());
out.writeInt(this.compressedWorldCompressionModeByteArray.size());
out.writeBytes(new ByteArrayInputStream(this.compressedWorldCompressionModeByteArray.elements()), this.compressedWorldCompressionModeByteArray.size());
out.writeInt(this.compressedMappingByteArray.size());
out.writeBytes(new ByteArrayInputStream(this.compressedMappingByteArray.elements()), this.compressedMappingByteArray.size());
out.writeByte(this.dataFormatVersion);
out.writeByte(this.compressionModeValue);
out.writeBoolean(this.applyToParent);
out.writeLong(this.lastModifiedUnixDateTime);
out.writeLong(this.createdUnixDateTime);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
@Override
@@ -400,16 +394,16 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
this.pos = in.readLong();
this.dataChecksum = in.readInt();
this.compressedDataByteArray = new byte[in.readInt()];
in.readBytes(this.compressedDataByteArray);
ListUtil.clearAndSetSize(this.compressedDataByteArray, in.readInt());
in.readBytes(this.compressedDataByteArray.elements());
this.compressedColumnGenStepByteArray = new byte[in.readInt()];
in.readBytes(this.compressedColumnGenStepByteArray);
this.compressedWorldCompressionModeByteArray = new byte[in.readInt()];
in.readBytes(this.compressedWorldCompressionModeByteArray);
ListUtil.clearAndSetSize(this.compressedColumnGenStepByteArray, in.readInt());
in.readBytes(this.compressedColumnGenStepByteArray.elements());
ListUtil.clearAndSetSize(this.compressedWorldCompressionModeByteArray, in.readInt());
in.readBytes(this.compressedWorldCompressionModeByteArray.elements());
this.compressedMappingByteArray = new byte[in.readInt()];
in.readBytes(this.compressedMappingByteArray);
ListUtil.clearAndSetSize(this.compressedMappingByteArray, in.readInt());
in.readBytes(this.compressedMappingByteArray.elements());
this.dataFormatVersion = in.readByte();
this.compressionModeValue = in.readByte();
@@ -422,6 +416,14 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
//================//
// helper methods //
//================//
public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); }
//===========//
// overrides //
//===========//
@@ -438,10 +440,10 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
.add("levelMinY", this.levelMinY)
.add("pos", this.pos)
.add("dataChecksum", this.dataChecksum)
.add("compressedDataByteArray length", this.compressedDataByteArray.length)
.add("compressedColumnGenStepByteArray length", this.compressedColumnGenStepByteArray.length)
.add("compressedWorldCompressionModeByteArray length", this.compressedWorldCompressionModeByteArray.length)
.add("compressedMappingByteArray length", this.compressedMappingByteArray.length)
.add("compressedDataByteArray length", this.compressedDataByteArray.size())
.add("compressedColumnGenStepByteArray length", this.compressedColumnGenStepByteArray.size())
.add("compressedWorldCompressionModeByteArray length", this.compressedWorldCompressionModeByteArray.size())
.add("compressedMappingByteArray length", this.compressedMappingByteArray.size())
.add("dataFormatVersion", this.dataFormatVersion)
.add("compressionModeValue", this.compressionModeValue)
.add("applyToParent", this.applyToParent)
@@ -450,10 +452,6 @@ public class FullDataSourceV2DTO implements IBaseDTO<Long>, INetworkObject
.toString();
}
//================//
// helper methods //
//================//
public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); }
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java
+4 -1
View File
@@ -23,11 +23,14 @@ package com.seibel.distanthorizons.core.sql.dto;
* DTO = DataTable Object <br>
* Any object that's stored in the database should extend this object.
*/
public interface IBaseDTO<TKey>
public interface IBaseDTO<TKey> extends AutoCloseable
{
TKey getKey();
/** Can be used for keys that don't have a clean human readable toString() method. */
default String getKeyDisplayString() { return this.getKey().toString(); }
// closing a DTO shouldn't have the possibility of throwing anything
@Override
void close();
}
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java
+216 -114
View File
@@ -20,6 +20,7 @@
package com.seibel.distanthorizons.core.sql.repo;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.DatabaseUpdater;
import com.seibel.distanthorizons.core.sql.DbConnectionClosedException;
import com.seibel.distanthorizons.core.sql.dto.IBaseDTO;
@@ -179,13 +180,20 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
public TDTO getByKey(TKey primaryKey)
{
Map<String, Object> objectMap = this.queryDictionaryFirst(this.createSelectByKeySql(primaryKey));
if (objectMap != null && !objectMap.isEmpty())
try(ResultSet resultSet = this.query(this.createSelectStatementByKey(primaryKey)))
{
return this.convertDictionaryToDto(objectMap);
if (resultSet != null && resultSet.next())
{
return this.convertResultSetToDto(resultSet);
}
else
{
return null;
}
}
else
catch (SQLException | IOException e)
{
LOGGER.warn("Unexpected issue deserializing DTO ["+this.dtoClass.getSimpleName()+"] with primary key ["+primaryKey+"]. Error: ["+e.getMessage()+"].", e);
return null;
}
}
@@ -228,7 +236,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
}
catch (SQLException e)
{
String message = "Unexpected insert statement error: ["+e.getMessage()+"].";
String message = "Unexpected DTO insert error: ["+e.getMessage()+"].";
LOGGER.error(message);
throw new RuntimeException(message, e);
}
@@ -245,7 +253,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
}
catch (SQLException e)
{
String message = "Unexpected update statement error: ["+e.getMessage()+"].";
String message = "Unexpected DTO update error: ["+e.getMessage()+"].";
LOGGER.error(message);
throw new RuntimeException(message, e);
}
@@ -255,55 +263,67 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
public void delete(TDTO dto) { this.deleteWithKey(dto.getKey()); }
public void deleteWithKey(TKey key)
{
String whereEqualStatement = this.createWhereStatement(key);
this.queryDictionaryFirst("DELETE FROM "+this.getTableName()+" WHERE "+whereEqualStatement);
try (PreparedStatement statement = this.createDeleteStatementByKey(key))
{
this.query(statement);
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
}
/** With great power comes great responsibility... */
public void deleteAll() { this.queryDictionaryFirst("DELETE FROM "+this.getTableName()); }
public void deleteAll()
{
String sql = "DELETE FROM " + this.getTableName();
try (PreparedStatement statement = this.createPreparedStatement(sql))
{
this.query(statement);
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
}
public boolean exists(TDTO dto) { return this.existsWithKey(dto.getKey()); }
public boolean existsWithKey(TKey key)
{
String whereEqualStatement = this.createWhereStatement(key);
Map<String, Object> result = this.queryDictionaryFirst("SELECT EXISTS(SELECT 1 FROM "+this.getTableName()+" WHERE "+whereEqualStatement+") as 'existingCount';");
return result != null && (int)result.get("existingCount") != 0;
try
{
try (PreparedStatement statement = this.createExistsStatementByKey(key);
ResultSet result = this.query(statement))
{
return result != null && result.getInt("existingCount") != 0;
}
}
catch (SQLException e)
{
return false;
}
}
//==============//
// low level DB //
//==============//
public List<Map<String, Object>> queryDictionary(String sql)
{
try
{
return this.query(sql);
}
catch (DbConnectionClosedException e)
{
return new ArrayList<>();
}
}
public List<Map<String, Object>> queryDictionary(PreparedStatement preparedStatement)
{
try
{
return this.query(preparedStatement);
}
catch (DbConnectionClosedException e)
{
return new ArrayList<>();
}
}
/**
* This can only run 1 command at a time. <br><br>
*
* Note: {@link AbstractDhRepo#query(PreparedStatement)} with a {@link PreparedStatement}
* should be used if the query will be run often.
* This reduces GC pressure due to the {@link String} and {@link Map} allocation cost.
*/
@Nullable
public Map<String, Object> queryDictionaryFirst(String sql)
{
try
{
List<Map<String, Object>> objectList = this.query(sql);
List<Map<String, Object>> objectList = this.queryDictionary(sql);
return !objectList.isEmpty() ? objectList.get(0) : null;
}
catch (DbConnectionClosedException e)
@@ -311,54 +331,14 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
return null;
}
}
@Nullable
public Map<String, Object> queryDictionaryFirst(PreparedStatement preparedStatement)
{
try
{
List<Map<String, Object>> objectList = this.query(preparedStatement);
return !objectList.isEmpty() ? objectList.get(0) : null;
}
catch (DbConnectionClosedException e)
{
return null;
}
}
/** note: this can only handle 1 command at a time */
private List<Map<String, Object>> query(PreparedStatement statement) throws RuntimeException, DbConnectionClosedException
{
try
{
statement.setQueryTimeout(TIMEOUT_SECONDS);
// Note: this can only handle 1 command at a time
boolean resultSetPresent = statement.execute();
try (ResultSet resultSet = statement.getResultSet())
{
return this.parseQueryResult(resultSet, resultSetPresent);
}
}
catch(SQLException e)
{
// SQL exceptions generally only happen when something is wrong with
// the database or the query and should cause the system to blow up to notify the developer
if (DbConnectionClosedException.IsClosedException(e))
{
throw new DbConnectionClosedException(e);
}
else
{
String message = "Unexpected Query error: [" + e.getMessage() + "], for prepared statement: [" + statement + "].";
LOGGER.error(message);
throw new RuntimeException(message, e);
}
}
}
/** note: this can only handle 1 command at a time */
private List<Map<String, Object>> query(String sql) throws RuntimeException, DbConnectionClosedException
/**
* This can only run 1 command at a time. <br><br>
*
* Note: {@link AbstractDhRepo#query(PreparedStatement)} with a {@link PreparedStatement}
* should be used if the query will be run often.
* This reduces GC pressure due to the {@link String} and {@link Map} allocation cost.
*/
private List<Map<String, Object>> queryDictionary(String sql) throws RuntimeException, DbConnectionClosedException
{
try (Statement statement = this.connection.createStatement())
{
@@ -368,7 +348,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
boolean resultSetPresent = statement.execute(sql);
try (ResultSet resultSet = statement.getResultSet())
{
return this.parseQueryResult(resultSet, resultSetPresent);
return this.convertResultSetToDictionaryList(resultSet, resultSetPresent);
}
}
catch(SQLException e)
@@ -383,32 +363,71 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
else
{
String message = "Unexpected Query error: [" + e.getMessage() + "], for script: [" + sql + "].";
LOGGER.error(message, e);
throw new RuntimeException(message, e);
}
}
}
/**
* Warning: both the returned {@link ResultSet} and incoming {@link PreparedStatement}
* must be wrapped in a try-resource block to prevent memory
* leaks and issues with the DB becoming locked.
*/
@Nullable
public ResultSet query(@Nullable PreparedStatement statement) throws RuntimeException
{
// This is done so we don't have to add "if null" checks everywhere.
// Normally this should only happen once the DB has been closed.
if (statement == null)
{
return null;
}
try
{
statement.setQueryTimeout(TIMEOUT_SECONDS);
// Note: this can only handle 1 command at a time
boolean resultSetPresent = statement.execute();
if (resultSetPresent)
{
return statement.getResultSet();
}
else
{
return null;
}
}
catch(SQLException e)
{
// SQL exceptions generally only happen when something is wrong with
// the database or the query and should cause the system to blow up to notify the developer
if (DbConnectionClosedException.IsClosedException(e))
{
return null;
}
else
{
String message = "Unexpected Query error: [" + e.getMessage() + "], for prepared statement: [" + statement + "].";
LOGGER.error(message);
throw new RuntimeException(message, e);
}
}
}
private List<Map<String, Object>> parseQueryResult(ResultSet resultSet, boolean resultSetPresent) throws SQLException
{
if (resultSetPresent)
{
List<Map<String, Object>> resultList = convertResultSetToDictionaryList(resultSet);
resultSet.close();
return resultList;
}
else
{
if (resultSet != null)
{
resultSet.close();
}
return new ArrayList<>();
}
}
public PreparedStatement createPreparedStatement(String sql) throws DbConnectionClosedException
/**
* @return Null if the database was closed
* @throws RuntimeException if there was a problem with the given SQL string
*/
@Nullable
public PreparedStatement createPreparedStatement(String sql) throws RuntimeException
{
try
{
@@ -420,7 +439,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
{
if (DbConnectionClosedException.IsClosedException(e))
{
throw new DbConnectionClosedException(e);
return null;
}
else
{
@@ -525,9 +544,25 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
// helper methods //
//================//
public String createWhereStatement(TDTO dto) { return this.createWhereStatement(dto.getKey()); }
public static List<Map<String, Object>> convertResultSetToDictionaryList(ResultSet resultSet) throws SQLException
private List<Map<String, Object>> convertResultSetToDictionaryList(ResultSet resultSet, boolean resultSetPresent) throws SQLException
{
if (resultSetPresent)
{
List<Map<String, Object>> resultList = convertResultSetToDictionaryList(resultSet);
resultSet.close();
return resultList;
}
else
{
if (resultSet != null)
{
resultSet.close();
}
return new ArrayList<>();
}
}
private static List<Map<String, Object>> convertResultSetToDictionaryList(ResultSet resultSet) throws SQLException
{
List<Map<String, Object>> list = new ArrayList<>();
@@ -540,7 +575,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
for (int columnIndex = 1; columnIndex <= resultColumnCount; columnIndex++) // column indices start at 1
{
String columnName = resultMetaData.getColumnName(columnIndex);
if (columnName == null || columnName.equals(""))
if (columnName == null || columnName.isEmpty())
{
throw new RuntimeException("SQL result set is missing a column name for column ["+resultMetaData.getTableName(columnIndex)+"."+columnIndex+"].");
}
@@ -585,18 +620,85 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
public abstract String getTableName();
@Nullable
public abstract TDTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException;
public abstract TDTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, IOException, SQLException;
public String createSelectByKeySql(TKey key) { return "SELECT * FROM "+this.getTableName()+" WHERE "+this.createWhereStatement(key); }
/**
* Example:
* <code> Id = '0' </code>
* <code> ColOne = '0' AND ColTwo = '2' </code>
* should NOT start with WHERE
* Example: TODO
*/
public abstract String createWhereStatement(TKey key);
protected abstract String CreateParameterizedWhereString();
protected void setPreparedStatementWhereClause(PreparedStatement statement, TKey key) throws SQLException { this.setPreparedStatementWhereClause(statement, 1, key); }
protected abstract int setPreparedStatementWhereClause(PreparedStatement statement, int parameterIndex, TKey key) throws SQLException;
private String selectSqlTemplate = null;
public PreparedStatement createSelectStatementByKey(TKey key) throws SQLException
{
// create shared template string
if (this.selectSqlTemplate == null)
{
this.selectSqlTemplate = "SELECT * FROM "+this.getTableName() + " WHERE " + this.CreateParameterizedWhereString();
}
PreparedStatement statement = this.createPreparedStatement(this.selectSqlTemplate);
if (statement == null)
{
return null;
}
this.setPreparedStatementWhereClause(statement, key);
return statement;
}
private String existsSqlTemplate = null;
public PreparedStatement createExistsStatementByKey(TKey key) throws SQLException
{
// create shared template string
if (this.existsSqlTemplate == null)
{
this.existsSqlTemplate = "SELECT EXISTS(SELECT 1 FROM "+this.getTableName()+" WHERE "+this.CreateParameterizedWhereString()+") as 'existingCount'";
}
PreparedStatement statement = this.createPreparedStatement(this.existsSqlTemplate);
if (statement == null)
{
return null;
}
this.setPreparedStatementWhereClause(statement, key);
return statement;
}
private String deleteSqlTemplate = null;
public PreparedStatement createDeleteStatementByKey(TKey key) throws SQLException
{
// create shared template string
if (this.deleteSqlTemplate == null)
{
this.deleteSqlTemplate = "DELETE FROM "+this.getTableName()+" WHERE " + this.CreateParameterizedWhereString();
}
PreparedStatement statement = this.createPreparedStatement(this.deleteSqlTemplate);
if (statement == null)
{
return null;
}
this.setPreparedStatementWhereClause(statement, key);
return statement;
}
@Nullable
public abstract PreparedStatement createInsertStatement(TDTO dto) throws SQLException;
@Nullable
public abstract PreparedStatement createUpdateStatement(TDTO dto) throws SQLException;
}
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/BeaconBeamRepo.java
+75 -32
View File
@@ -26,10 +26,12 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO;
import com.seibel.distanthorizons.core.util.LodUtil;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.File;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
@@ -60,7 +62,16 @@ public class BeaconBeamRepo extends AbstractDhRepo<DhBlockPos, BeaconBeamDTO>
public String getTableName() { return "BeaconBeam"; }
@Override
public String createWhereStatement(DhBlockPos pos) { return "BlockPosX = "+ pos.getX() +" AND BlockPosY = "+ pos.getY() +" AND BlockPosZ = "+ pos.getZ(); }
protected String CreateParameterizedWhereString() { return "BlockPosX = ? AND BlockPosY = ? AND BlockPosZ = ?"; }
@Override
protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, DhBlockPos pos) throws SQLException
{
statement.setInt(index++, pos.getX());
statement.setInt(index++, pos.getY());
statement.setInt(index++, pos.getZ());
return index;
}
@@ -69,15 +80,16 @@ public class BeaconBeamRepo extends AbstractDhRepo<DhBlockPos, BeaconBeamDTO>
//=======================//
@Override
public BeaconBeamDTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
@Nullable
public BeaconBeamDTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, SQLException
{
int posX = (Integer) objectMap.get("BlockPosX");
int posY = (Integer) objectMap.get("BlockPosY");
int posZ = (Integer) objectMap.get("BlockPosZ");
int posX = resultSet.getInt("BlockPosX");
int posY = resultSet.getInt("BlockPosY");
int posZ = resultSet.getInt("BlockPosZ");
int red = (Integer) objectMap.get("ColorR");
int green = (Integer) objectMap.get("ColorG");
int blue = (Integer) objectMap.get("ColorB");
int red = resultSet.getInt("ColorR");
int green = resultSet.getInt("ColorG");
int blue = resultSet.getInt("ColorB");
BeaconBeamDTO dto = new BeaconBeamDTO(new DhBlockPos(posX, posY, posZ), new Color(red, green, blue));
@@ -98,18 +110,23 @@ public class BeaconBeamRepo extends AbstractDhRepo<DhBlockPos, BeaconBeamDTO>
" ?, ? \n" +
");";
PreparedStatement statement = this.createPreparedStatement(sql);
if (statement == null)
{
return null;
}
int i = 1;
statement.setObject(i++, dto.blockPos.getX());
statement.setObject(i++, dto.blockPos.getY());
statement.setObject(i++, dto.blockPos.getZ());
statement.setInt(i++, dto.blockPos.getX());
statement.setInt(i++, dto.blockPos.getY());
statement.setInt(i++, dto.blockPos.getZ());
statement.setObject(i++, dto.color.getRed());
statement.setObject(i++, dto.color.getGreen());
statement.setObject(i++, dto.color.getBlue());
statement.setInt(i++, dto.color.getRed());
statement.setInt(i++, dto.color.getGreen());
statement.setInt(i++, dto.color.getBlue());
statement.setObject(i++, System.currentTimeMillis()); // last modified unix time
statement.setObject(i++, System.currentTimeMillis()); // created unix time
statement.setLong(i++, System.currentTimeMillis()); // last modified unix time
statement.setLong(i++, System.currentTimeMillis()); // created unix time
return statement;
}
@@ -124,17 +141,21 @@ public class BeaconBeamRepo extends AbstractDhRepo<DhBlockPos, BeaconBeamDTO>
" LastModifiedUnixDateTime = ? \n" +
"WHERE BlockPosX = ? AND BlockPosY = ? AND BlockPosZ = ?";
PreparedStatement statement = this.createPreparedStatement(sql);
if (statement == null)
{
return null;
}
int i = 1;
statement.setObject(i++, dto.color.getRed());
statement.setObject(i++, dto.color.getGreen());
statement.setObject(i++, dto.color.getBlue());
statement.setInt(i++, dto.color.getRed());
statement.setInt(i++, dto.color.getGreen());
statement.setInt(i++, dto.color.getBlue());
statement.setObject(i++, System.currentTimeMillis()); // last modified unix time
statement.setLong(i++, System.currentTimeMillis()); // last modified unix time
statement.setObject(i++, dto.blockPos.getX());
statement.setObject(i++, dto.blockPos.getY());
statement.setObject(i++, dto.blockPos.getZ());
statement.setInt(i++, dto.blockPos.getX());
statement.setInt(i++, dto.blockPos.getY());
statement.setInt(i++, dto.blockPos.getZ());
return statement;
}
@@ -171,22 +192,44 @@ public class BeaconBeamRepo extends AbstractDhRepo<DhBlockPos, BeaconBeamDTO>
);
}
private final String getAllBeamsInRangeTemplate =
"SELECT * " +
"FROM "+this.getTableName()+" " +
"WHERE " +
"? <= BlockPosX AND BlockPosX <= ? AND " +
"? <= BlockPosZ AND BlockPosZ <= ?";
public List<BeaconBeamDTO> getAllBeamsInBlockPosRange(
int minBlockX, int minBlockZ,
int maxBlockX, int maxBlockZ
)
{
List<Map<String, Object>> objectMapList = this.queryDictionary(
"SELECT * " +
"FROM "+this.getTableName()+" " +
"WHERE " +
minBlockX+" <= BlockPosX AND BlockPosX <= "+maxBlockX+" AND " +
minBlockZ+" <= BlockPosZ AND BlockPosZ <= "+maxBlockZ);
ArrayList<BeaconBeamDTO> beamList = new ArrayList<>();
for (Map<String, Object> objectMap : objectMapList)
try(PreparedStatement statement = this.createPreparedStatement(this.getAllBeamsInRangeTemplate))
{
beamList.add(this.convertDictionaryToDto(objectMap));
if(statement == null)
{
return beamList;
}
int i = 1;
statement.setInt(i++, minBlockX);
statement.setInt(i++, minBlockZ);
statement.setInt(i++, maxBlockX);
statement.setInt(i++, maxBlockZ);
try (ResultSet result = this.query(statement))
{
while (result != null && result.next())
{
beamList.add(this.convertResultSetToDto(result));
}
}
}
catch (Exception e)
{
throw new RuntimeException(e);
}
return beamList;
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/ChunkHashRepo.java
+18 -6
View File
@@ -21,11 +21,14 @@ package com.seibel.distanthorizons.core.sql.repo;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO;
import com.seibel.distanthorizons.core.sql.dto.ChunkHashDTO;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;
@@ -54,7 +57,15 @@ public class ChunkHashRepo extends AbstractDhRepo<DhChunkPos, ChunkHashDTO>
public String getTableName() { return "ChunkHash"; }
@Override
public String createWhereStatement(DhChunkPos pos) { return "ChunkPosX = '"+ pos.getX() +"' AND ChunkPosZ = '"+ pos.getZ() +"'"; }
protected String CreateParameterizedWhereString() { return "ChunkPosX = ? AND ChunkPosZ = ?"; }
@Override
protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, DhChunkPos pos) throws SQLException
{
statement.setInt(index++, pos.getX());
statement.setInt(index++, pos.getZ());
return index;
}
@@ -62,13 +73,14 @@ public class ChunkHashRepo extends AbstractDhRepo<DhChunkPos, ChunkHashDTO>
// repo required methods //
//=======================//
@Override
public ChunkHashDTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
@Override
@Nullable
public ChunkHashDTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, SQLException
{
int posX = (Integer) objectMap.get("ChunkPosX");
int posZ = (Integer) objectMap.get("ChunkPosZ");
int posX = resultSet.getInt("ChunkPosX");
int posZ = resultSet.getInt("ChunkPosZ");
int chunkHash = (Integer) objectMap.get("ChunkHash");
int chunkHash = resultSet.getInt("ChunkHash");
ChunkHashDTO dto = new ChunkHashDTO(new DhChunkPos(posX, posZ), chunkHash);
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java
+124 -86
View File
@@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.sql.repo;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.ChunkHashDTO;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import com.seibel.distanthorizons.coreapi.util.StringUtil;
import it.unimi.dsi.fastutil.longs.LongArrayList;
@@ -28,6 +29,7 @@ import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
@@ -58,7 +60,14 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
public String getTableName() { return TABLE_NAME; }
@Override
public String createWhereStatement(Long pos) { return "DhSectionPos = '"+serializeSectionPos(pos)+"'"; }
protected String CreateParameterizedWhereString() { return "DhSectionPos = ?"; }
@Override
protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, Long pos) throws SQLException
{
statement.setString(index++, serializeSectionPos(pos));
return index;
}
@@ -66,23 +75,24 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
// repo required methods //
//=======================//
@Override
public FullDataSourceV1DTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
@Override
@Nullable
public FullDataSourceV1DTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, SQLException
{
String posString = (String) objectMap.get("DhSectionPos");
String posString = resultSet.getString("DhSectionPos");
Long pos = deserializeSectionPos(posString);
// meta data
int checksum = (Integer) objectMap.get("Checksum");
byte dataDetailLevel = (Byte) objectMap.get("DataDetailLevel");
String worldGenStepString = (String) objectMap.get("WorldGenStep");
int checksum = resultSet.getInt("Checksum");
byte dataDetailLevel = resultSet.getByte("DataDetailLevel");
String worldGenStepString = resultSet.getString("WorldGenStep");
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromName(worldGenStepString);
String dataType = (String) objectMap.get("DataType");
byte binaryDataFormatVersion = (Byte) objectMap.get("BinaryDataFormatVersion");
String dataType = resultSet.getString("DataType");
byte binaryDataFormatVersion = resultSet.getByte("BinaryDataFormatVersion");
// binary data
byte[] dataByteArray = (byte[]) objectMap.get("Data");
byte[] dataByteArray = resultSet.getBytes("Data");
FullDataSourceV1DTO dto = new FullDataSourceV1DTO(
pos,
@@ -92,41 +102,43 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
return dto;
}
private final String insertSqlTemplate =
"INSERT INTO "+this.getTableName() + "\n" +
" (DhSectionPos, \n" +
"Checksum, DataVersion, DataDetailLevel, WorldGenStep, DataType, BinaryDataFormatVersion, \n" +
"Data) \n" +
" VALUES( \n" +
" ? \n" +
" ,? ,? ,? ,? ,? ,? \n" +
" ,? \n" +
// created/lastModified are automatically set by Sqlite
");";
@Override
public PreparedStatement createInsertStatement(FullDataSourceV1DTO dto) throws SQLException
{
String sql =
"INSERT INTO "+this.getTableName() + "\n" +
" (DhSectionPos, \n" +
"Checksum, DataVersion, DataDetailLevel, WorldGenStep, DataType, BinaryDataFormatVersion, \n" +
"Data) \n" +
" VALUES( \n" +
" ? \n" +
" ,? ,? ,? ,? ,? ,? \n" +
" ,? \n" +
// created/lastModified are automatically set by Sqlite
");";
PreparedStatement statement = this.createPreparedStatement(sql);
PreparedStatement statement = this.createPreparedStatement(this.insertSqlTemplate);
if (statement == null)
{
return null;
}
int i = 1;
statement.setObject(i++, serializeSectionPos(dto.pos));
statement.setString(i++, serializeSectionPos(dto.pos));
statement.setObject(i++, dto.checksum);
statement.setObject(i++, 0 /*dto.dataVersion*/);
statement.setObject(i++, dto.dataDetailLevel);
statement.setInt(i++, dto.checksum);
statement.setInt(i++, 0 /*dto.dataVersion*/);
statement.setByte(i++, dto.dataDetailLevel);
statement.setObject(i++, dto.worldGenStep);
statement.setObject(i++, dto.dataType);
statement.setObject(i++, dto.binaryDataFormatVersion);
statement.setString(i++, dto.dataType);
statement.setByte(i++, dto.binaryDataFormatVersion);
statement.setObject(i++, dto.dataArray);
return statement;
}
@Override
public PreparedStatement createUpdateStatement(FullDataSourceV1DTO dto) throws SQLException
{
String sql =
private final String updateSqlTemplate =
"UPDATE "+this.getTableName()+" \n" +
"SET \n" +
" Checksum = ? \n" +
@@ -135,56 +147,38 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
" ,WorldGenStep = ? \n" +
" ,DataType = ? \n" +
" ,BinaryDataFormatVersion = ? \n" +
" ,Data = ? \n" +
" ,LastModifiedDateTime = CURRENT_TIMESTAMP \n" +
"WHERE DhSectionPos = ?";
PreparedStatement statement = this.createPreparedStatement(sql);
@Override
public PreparedStatement createUpdateStatement(FullDataSourceV1DTO dto) throws SQLException
{
PreparedStatement statement = this.createPreparedStatement(this.updateSqlTemplate);
if (statement == null)
{
return null;
}
int i = 1;
statement.setObject(i++, dto.checksum);
statement.setObject(i++, 0 /*dto.dataVersion*/);
statement.setObject(i++, dto.dataDetailLevel);
statement.setInt(i++, dto.checksum);
statement.setInt(i++, 0 /*dto.dataVersion*/);
statement.setByte(i++, dto.dataDetailLevel);
statement.setObject(i++, dto.worldGenStep);
statement.setObject(i++, dto.dataType);
statement.setObject(i++, dto.binaryDataFormatVersion);
statement.setString(i++, dto.dataType);
statement.setByte(i++, dto.binaryDataFormatVersion);
statement.setObject(i++, dto.dataArray);
statement.setObject(i++, serializeSectionPos(dto.pos));
statement.setString(i++, serializeSectionPos(dto.pos));
return statement;
}
//=====================//
// data source methods //
//=====================//
/**
* Returns the highest numerical detail level in this table. <Br>
* Returns {@link DhSectionPos#SECTION_MINIMUM_DETAIL_LEVEL} if no data is present.
*/
public int getMaxSectionDetailLevel()
{
Map<String, Object> resultMap = this.queryDictionaryFirst("select MAX(DataDetailLevel) as maxDetailLevel from "+this.getTableName()+";");
int maxDetailLevel;
if (resultMap == null || resultMap.get("maxDetailLevel") == null)
{
maxDetailLevel = 0;
}
else
{
maxDetailLevel = (int)resultMap.get("maxDetailLevel");
}
return maxDetailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
}
//===========//
// migration //
//===========//
@@ -207,25 +201,47 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
}
}
private final String getMigrationPositionsSqlTemplate =
"SELECT DhSectionPos " +
"FROM "+this.getTableName()+" " +
"WHERE MigrationFailed <> 1 " +
"LIMIT ?;";
/** Returns the new "returnCount" positions that need to be migrated */
public LongArrayList getPositionsToMigrate(int returnCount)
{
LongArrayList list = new LongArrayList();
LongArrayList posList = new LongArrayList();
List<Map<String, Object>> resultMapList = this.queryDictionary(
"select DhSectionPos " +
"from "+this.getTableName()+" " +
"WHERE MigrationFailed <> 1 " +
"LIMIT "+returnCount+";");
for (Map<String, Object> resultMap : resultMapList)
try(PreparedStatement statement = this.createPreparedStatement(this.getMigrationPositionsSqlTemplate))
{
// returned in the format [sectionDetailLevel,x,z] IE [6,0,0]
long sectionPos = deserializeSectionPos((String) resultMap.get("DhSectionPos"));
list.add(sectionPos);
if (statement == null)
{
return posList;
}
int i = 1;
statement.setInt(i++, returnCount);
try (ResultSet result = this.query(statement))
{
while (result != null && result.next())
{
String posString = result.getString("DhSectionPos");
// returned in the format [sectionDetailLevel,x,z] IE [6,0,0]
Long sectionPos = deserializeSectionPos(posString);
if (sectionPos != null)
{
posList.add(sectionPos.longValue());
}
}
}
}
catch (Exception e)
{
throw new RuntimeException(e);
}
return list;
return posList;
}
public void markMigrationFailed(long pos)
@@ -263,17 +279,38 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
}
}
private final String getUnusedPositionSqlTemplate =
"SELECT DhSectionPos " +
"FROM "+this.getTableName()+" " +
"WHERE DataDetailLevel <> 0 OR DataType <> 'CompleteFullDataSource' " +
"LIMIT ?";
/** Returns single quote surrounded {@link DhSectionPos} serailzed values */
public ArrayList<String> getUnusedDataSourcePositionStringList(int deleteCount)
public ArrayList<String> getUnusedDataSourcePositionStringList(int limit)
{
List<Map<String, Object>> deletePosResultMapList = this.queryDictionary(
"select DhSectionPos from "+this.getTableName()+" where DataDetailLevel <> 0 or DataType <> 'CompleteFullDataSource' limit "+deleteCount);
ArrayList<String> deletePosList = new ArrayList<>();
for (Map<String, Object> deletePosMap : deletePosResultMapList)
try(PreparedStatement statement = this.createPreparedStatement(this.getUnusedPositionSqlTemplate))
{
String posString = (String) deletePosMap.get("DhSectionPos");
deletePosList.add("'"+posString+"'");
if (statement == null)
{
return deletePosList;
}
int i = 1;
statement.setInt(i++, limit);
try (ResultSet result = this.query(statement))
{
while (result != null && result.next())
{
String posString = result.getString("DhSectionPos");
deletePosList.add("'"+posString+"'");
}
}
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
return deletePosList;
@@ -294,7 +331,6 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
private static String serializeSectionPos(long pos) { return "[" + DhSectionPos.getDetailLevel(pos) + ',' + DhSectionPos.getX(pos) + ',' + DhSectionPos.getZ(pos) + ']'; }
@Nullable
private static Long deserializeSectionPos(String value)
{
@@ -312,4 +348,6 @@ public class FullDataSourceV1Repo extends AbstractDhRepo<Long, FullDataSourceV1D
return DhSectionPos.encode(Byte.parseByte(split[0]), Integer.parseInt(split[1]), Integer.parseInt(split[2]));
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
+339 -174
View File
@@ -26,19 +26,17 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.DbConnectionClosedException;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.*;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2DTO>
{
@@ -65,71 +63,98 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
public String getTableName() { return "FullData"; }
@Override
public String createWhereStatement(Long pos)
protected String CreateParameterizedWhereString() { return "DetailLevel = ? AND PosX = ? AND PosZ = ?"; }
@Override
protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, Long pos) throws SQLException
{
int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL;
return "DetailLevel = '"+detailLevel+"' AND PosX = '"+ DhSectionPos.getX(pos)+"' AND PosZ = '"+ DhSectionPos.getZ(pos)+"'";
statement.setInt(index++, detailLevel);
statement.setInt(index++, DhSectionPos.getX(pos));
statement.setInt(index++, DhSectionPos.getZ(pos));
return index;
}
//=======================//
// repo required methods //
//=======================//
@Override
public FullDataSourceV2DTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
@Nullable
public FullDataSourceV2DTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, IOException, SQLException
{
byte detailLevel = (Byte) objectMap.get("DetailLevel");
//======================//
// get statement values //
//======================//
byte detailLevel = resultSet.getByte("DetailLevel");
byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
int posX = (Integer) objectMap.get("PosX");
int posZ = (Integer) objectMap.get("PosZ");
int posX = resultSet.getInt("PosX");
int posZ = resultSet.getInt("PosZ");
long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ);
int minY = (Integer) objectMap.get("MinY");
int dataChecksum = (Integer) objectMap.get("DataChecksum");
byte[] dataByteArray = (byte[]) objectMap.get("Data");
byte[] columnGenStepByteArray = (byte[]) objectMap.get("ColumnGenerationStep");
byte[] columnWorldCompressionByteArray = (byte[]) objectMap.get("ColumnWorldCompressionMode");
byte[] mappingByteArray = (byte[]) objectMap.get("Mapping");
int minY = resultSet.getInt("MinY");
int dataChecksum = resultSet.getInt("DataChecksum");
byte dataFormatVersion = (Byte) objectMap.get("DataFormatVersion");
byte compressionModeValue = (Byte) objectMap.get("CompressionMode");
byte dataFormatVersion = resultSet.getByte("DataFormatVersion");
byte compressionModeValue = resultSet.getByte("CompressionMode");
boolean applyToParent = ((int) objectMap.get("ApplyToParent")) == 1;
boolean applyToParent = (resultSet.getInt("ApplyToParent")) == 1;
long lastModifiedUnixDateTime = (Long) objectMap.get("LastModifiedUnixDateTime");
long createdUnixDateTime = (Long) objectMap.get("CreatedUnixDateTime");
long lastModifiedUnixDateTime = resultSet.getLong("LastModifiedUnixDateTime");
long createdUnixDateTime = resultSet.getLong("CreatedUnixDateTime");
FullDataSourceV2DTO dto = new FullDataSourceV2DTO(
pos,
dataChecksum, columnGenStepByteArray, columnWorldCompressionByteArray, dataFormatVersion, compressionModeValue, dataByteArray,
lastModifiedUnixDateTime, createdUnixDateTime,
mappingByteArray, applyToParent,
minY);
//===================//
// set DTO variables //
//===================//
FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding();
// set pooled arrays
dto.compressedDataByteArray = putAllBytes(resultSet.getBinaryStream("Data"), dto.compressedDataByteArray);
dto.compressedColumnGenStepByteArray = putAllBytes(resultSet.getBinaryStream("ColumnGenerationStep"), dto.compressedColumnGenStepByteArray);
dto.compressedWorldCompressionModeByteArray = putAllBytes(resultSet.getBinaryStream("ColumnWorldCompressionMode"), dto.compressedWorldCompressionModeByteArray);
dto.compressedMappingByteArray = putAllBytes(resultSet.getBinaryStream("Mapping"), dto.compressedMappingByteArray);
// set individual variables
{
dto.pos = pos;
dto.dataChecksum = dataChecksum;
dto.dataFormatVersion = dataFormatVersion;
dto.compressionModeValue = compressionModeValue;
dto.lastModifiedUnixDateTime = lastModifiedUnixDateTime;
dto.createdUnixDateTime = createdUnixDateTime;
dto.applyToParent = applyToParent;
dto.levelMinY = minY;
}
return dto;
}
private final String insertSqlTemplate =
"INSERT INTO "+this.getTableName() + " (\n" +
" DetailLevel, PosX, PosZ, \n" +
" MinY, DataChecksum, \n" +
" Data, ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" +
" DataFormatVersion, CompressionMode, ApplyToParent, \n" +
" LastModifiedUnixDateTime, CreatedUnixDateTime) \n" +
"VALUES( \n" +
" ?, ?, ?, \n" +
" ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ?, ?, \n" +
" ?, ? \n" +
");";
@Override
public PreparedStatement createInsertStatement(FullDataSourceV2DTO dto) throws SQLException
{
String sql =
"INSERT INTO "+this.getTableName() + " (\n" +
" DetailLevel, PosX, PosZ, \n" +
" MinY, DataChecksum, \n" +
" Data, ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" +
" DataFormatVersion, CompressionMode, ApplyToParent, \n" +
" LastModifiedUnixDateTime, CreatedUnixDateTime) \n" +
"VALUES( \n" +
" ?, ?, ?, \n" +
" ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ?, ?, \n" +
" ?, ? \n" +
");";
PreparedStatement statement = this.createPreparedStatement(sql);
PreparedStatement statement = this.createPreparedStatement(this.insertSqlTemplate);
if (statement == null)
{
return null;
}
int i = 1;
statement.setObject(i++, DhSectionPos.getDetailLevel(dto.pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
@@ -139,10 +164,10 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
statement.setObject(i++, dto.levelMinY);
statement.setObject(i++, dto.dataChecksum);
statement.setObject(i++, dto.compressedDataByteArray);
statement.setObject(i++, dto.compressedColumnGenStepByteArray);
statement.setObject(i++, dto.compressedWorldCompressionModeByteArray);
statement.setObject(i++, dto.compressedMappingByteArray);
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedDataByteArray.elements()), dto.compressedDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedColumnGenStepByteArray.elements()), dto.compressedColumnGenStepByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedWorldCompressionModeByteArray.elements()), dto.compressedWorldCompressionModeByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedMappingByteArray.elements()), dto.compressedMappingByteArray.size());
statement.setObject(i++, dto.dataFormatVersion);
statement.setObject(i++, dto.compressionModeValue);
@@ -154,38 +179,43 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
return statement;
}
@Override
public PreparedStatement createUpdateStatement(FullDataSourceV2DTO dto) throws SQLException
{
String sql =
private final String updateSqlTemplate =
"UPDATE "+this.getTableName()+" \n" +
"SET \n" +
" MinY = ? \n" +
" ,DataChecksum = ? \n" +
" ,Data = ? \n" +
" ,ColumnGenerationStep = ? \n" +
" ,ColumnWorldCompressionMode = ? \n" +
" ,Mapping = ? \n" +
" ,DataFormatVersion = ? \n" +
" ,CompressionMode = ? \n" +
" ,ApplyToParent = ? \n" +
" ,LastModifiedUnixDateTime = ? \n" +
" ,CreatedUnixDateTime = ? \n" +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
PreparedStatement statement = this.createPreparedStatement(sql);
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
@Override
public PreparedStatement createUpdateStatement(FullDataSourceV2DTO dto) throws SQLException
{
PreparedStatement statement = this.createPreparedStatement(this.updateSqlTemplate);
if (statement == null)
{
return null;
}
int i = 1;
statement.setObject(i++, dto.levelMinY);
statement.setObject(i++, dto.dataChecksum);
statement.setObject(i++, dto.compressedDataByteArray);
statement.setObject(i++, dto.compressedColumnGenStepByteArray);
statement.setObject(i++, dto.compressedWorldCompressionModeByteArray);
statement.setObject(i++, dto.compressedMappingByteArray);
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedDataByteArray.elements()), dto.compressedDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedColumnGenStepByteArray.elements()), dto.compressedColumnGenStepByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedWorldCompressionModeByteArray.elements()), dto.compressedWorldCompressionModeByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedMappingByteArray.elements()), dto.compressedMappingByteArray.size());
statement.setObject(i++, dto.dataFormatVersion);
statement.setObject(i++, dto.compressionModeValue);
@@ -207,82 +237,134 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
// updates //
//=========//
public void setApplyToParent(long pos, boolean applyToParent) throws SQLException
{
int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
String sql =
private final String setApplyToParentSql =
"UPDATE "+this.getTableName()+" \n" +
"SET ApplyToParent = "+applyToParent+" \n" +
"WHERE DetailLevel = "+detailLevel+" AND PosX = "+ DhSectionPos.getX(pos)+" AND PosZ = "+ DhSectionPos.getZ(pos);
"SET ApplyToParent = ? \n" +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
public void setApplyToParent(long pos, boolean applyToParent)
{
PreparedStatement statement = this.createPreparedStatement(this.setApplyToParentSql);
if (statement == null)
{
return;
}
this.queryDictionaryFirst(sql);
try
{
int i = 1;
statement.setBoolean(i++, applyToParent);
int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
statement.setInt(i++, detailLevel);
statement.setInt(i++, DhSectionPos.getX(pos));
statement.setInt(i++, DhSectionPos.getZ(pos));
this.query(statement);
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
}
private final String getPositionsToUpdateSql =
"SELECT DetailLevel, PosX, PosZ, " +
" (sqrt(pow(PosX - ?, 2) + pow(PosZ - ?, 2))) AS Distance " +
"FROM "+this.getTableName()+" " +
"WHERE ApplyToParent = 1 " +
"ORDER BY Distance ASC " +
"LIMIT ?; ";
public LongArrayList getPositionsToUpdate(int targetBlockPosX, int targetBlockPosZ, int returnCount)
{
LongArrayList list = new LongArrayList();
List<Map<String, Object>> resultMapList = this.queryDictionary(
"SELECT DetailLevel, PosX, PosZ, " +
"(sqrt(pow(PosX - "+targetBlockPosX+", 2) + pow(PosZ - "+targetBlockPosZ+", 2))) AS Distance " +
"FROM "+this.getTableName()+" " +
"WHERE ApplyToParent = 1 " +
"ORDER BY Distance ASC " +
"LIMIT "+returnCount+"; "
);
for (Map<String, Object> resultMap : resultMapList)
PreparedStatement statement = this.createPreparedStatement(this.getPositionsToUpdateSql);
if (statement == null)
{
byte detailLevel = (Byte) resultMap.get("DetailLevel");
byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
int posX = (Integer) resultMap.get("PosX");
int posZ = (Integer) resultMap.get("PosZ");
long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ);
list.add(pos);
return list;
}
return list;
try
{
int i = 1;
statement.setInt(i++, targetBlockPosX);
statement.setInt(i++, targetBlockPosZ);
statement.setInt(i++, returnCount);
ResultSet result = this.query(statement);
while (result != null && result.next())
{
byte detailLevel = result.getByte("DetailLevel");
byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
int posX = result.getInt("PosX");
int posZ = result.getInt("PosZ");
long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ);
list.add(pos);
}
return list;
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
}
private final String getColumnGenerationStepSql =
"select ColumnGenerationStep, CompressionMode " +
"from "+this.getTableName()+" " +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
/** @return null if nothing exists for this position */
public byte[] getColumnGenerationStepForPos(long pos)
public void getColumnGenerationStepForPos(long pos, ByteArrayList outputByteArray)
{
int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
Map<String, Object> resultMap = this.queryDictionaryFirst(
"select ColumnGenerationStep, CompressionMode " +
"from "+this.getTableName()+" " +
"WHERE DetailLevel = "+detailLevel+" AND PosX = "+ DhSectionPos.getX(pos)+" AND PosZ = "+ DhSectionPos.getZ(pos));
if (resultMap != null)
PreparedStatement statement = this.createPreparedStatement(this.getColumnGenerationStepSql);
if (statement == null)
{
byte[] compressedByteArray = (byte[]) resultMap.get("ColumnGenerationStep");
return;
}
try
{
int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
byte compressionModeEnumValue = (byte) resultMap.get("CompressionMode");
int i = 1;
statement.setInt(i++, detailLevel);
statement.setInt(i++, DhSectionPos.getX(pos));
statement.setInt(i++, DhSectionPos.getZ(pos));
ResultSet result = this.query(statement);
if (result == null || !result.next())
{
return;
}
byte compressionModeEnumValue = result.getByte("CompressionMode");
EDhApiDataCompressionMode compressionModeEnum = EDhApiDataCompressionMode.getFromValue(compressionModeEnumValue);
try
{
// decompress the data
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedByteArray);
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
compressedIn.readFully(columnGenStepByteArray);
return columnGenStepByteArray;
DhDataInputStream compressedIn = new DhDataInputStream(result.getBinaryStream("ColumnGenerationStep"), compressionModeEnum);
putAllBytes(compressedIn, outputByteArray);
}
catch (IOException e)
{
LOGGER.warn("Decompression issue when getting column gen steps for pos: [" + DhSectionPos.toString(pos) + "]", e);
return null;
}
}
else
catch (SQLException e)
{
return null;
throw new RuntimeException(e);
}
}
@@ -292,26 +374,36 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
// multiplayer //
//=============//
private final String getTimestampForPosSql =
"SELECT LastModifiedUnixDateTime " +
"FROM " + this.getTableName() + " " +
"WHERE DetailLevel = ? " +
"AND PosX = ? " +
"AND PosZ = ?;";
@Nullable
public Long getTimestampForPos(long pos)
{
try
{
PreparedStatement preparedStatement = this.createPreparedStatement(
"SELECT LastModifiedUnixDateTime " +
"FROM " + this.getTableName() + " " +
"WHERE DetailLevel = ? " +
"AND PosX = ? " +
"AND PosZ = ?;"
);
PreparedStatement preparedStatement = this.createPreparedStatement(this.getTimestampForPosSql);
if (preparedStatement == null)
{
return null;
}
int i = 1;
preparedStatement.setInt(i++, DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
preparedStatement.setInt(i++, DhSectionPos.getX(pos));
preparedStatement.setInt(i++, DhSectionPos.getZ(pos));
List<Map<String, Object>> row = this.queryDictionary(preparedStatement);
return !row.isEmpty() ? (Long) row.get(0).get("LastModifiedUnixDateTime") : null;
ResultSet result = this.query(preparedStatement);
if (result == null || !result.next())
{
return null;
}
return result.getLong("LastModifiedUnixDateTime");
}
catch (DbConnectionClosedException e)
{
@@ -323,17 +415,22 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
}
}
private final String getTimestampForRangeSql =
"SELECT PosX, PosZ, LastModifiedUnixDateTime " +
"FROM " + this.getTableName() + " " +
"WHERE DetailLevel = ? " +
"AND PosX BETWEEN ? AND ? " +
"AND PosZ BETWEEN ? AND ?;";
public Map<Long, Long> getTimestampsForRange(byte detailLevel, int startPosX, int startPosZ, int endPosX, int endPosZ)
{
try
{
PreparedStatement preparedStatement = this.createPreparedStatement(
"SELECT PosX, PosZ, LastModifiedUnixDateTime " +
"FROM " + this.getTableName() + " " +
"WHERE DetailLevel = ? " +
"AND PosX BETWEEN ? AND ? " +
"AND PosZ BETWEEN ? AND ?;"
);
PreparedStatement preparedStatement = this.createPreparedStatement(this.getTimestampForRangeSql);
if (preparedStatement == null)
{
return new HashMap<>();
}
int i = 1;
preparedStatement.setInt(i++, detailLevel - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
@@ -342,15 +439,18 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
preparedStatement.setInt(i++, startPosZ);
preparedStatement.setInt(i++, endPosZ - 1);
return this.queryDictionary(preparedStatement)
.stream().collect(Collectors.toMap(
row -> DhSectionPos.encode(detailLevel, (int) row.get("PosX"), (int) row.get("PosZ")),
row -> (long) row.get("LastModifiedUnixDateTime"))
);
}
catch (DbConnectionClosedException e)
{
return new HashMap<>();
ResultSet result = this.query(preparedStatement);
HashMap<Long, Long> returnMap = new HashMap<>();
while (result != null && result.next())
{
long key = DhSectionPos.encode(detailLevel, result.getInt("PosX"), result.getInt("PosZ"));
long value = result.getLong("LastModifiedUnixDateTime");
returnMap.put(key, value);
}
return returnMap;
}
catch (SQLException e)
{
@@ -364,29 +464,48 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
// compression tests //
//===================//
private final String getAllPositionsSql =
"select DetailLevel, PosX, PosZ " +
"from "+this.getTableName()+"; ";
/** @return every position in this database */
public LongArrayList getAllPositions()
{
LongArrayList list = new LongArrayList();
List<Map<String, Object>> resultMapList = this.queryDictionary(
"select DetailLevel, PosX, PosZ " +
"from "+this.getTableName()+"; ");
for (Map<String, Object> resultMap : resultMapList)
PreparedStatement statement = this.createPreparedStatement(getAllPositionsSql);
if (statement == null)
{
byte detailLevel = (Byte) resultMap.get("DetailLevel");
byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
int posX = (Integer) resultMap.get("PosX");
int posZ = (Integer) resultMap.get("PosZ");
long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ);
list.add(pos);
return list;
}
try
{
ResultSet result = this.query(statement);
while (result != null && result.next())
{
byte detailLevel = result.getByte("DetailLevel");
byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
int posX = result.getInt("PosX");
int posZ = result.getInt("PosZ");
long pos = DhSectionPos.encode(sectionDetailLevel, posX, posZ);
list.add(pos);
}
return list;
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
return list;
}
private final String getDataSizeInBytesSql =
"select LENGTH(Data) as dataSize " +
"from "+this.getTableName()+" " +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
/**
* @return the size of the full data at the given position
* (doesn't include the size of the mapping or any other column)
@@ -394,45 +513,91 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
public long getDataSizeInBytes(long pos)
{
int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
Map<String, Object> resultMap = this.queryDictionaryFirst(
"select LENGTH(Data) as dataSize " +
"from "+this.getTableName()+" " +
"WHERE DetailLevel = "+detailLevel+" AND PosX = "+ DhSectionPos.getX(pos)+" AND PosZ = "+ DhSectionPos.getZ(pos));
if (resultMap != null && resultMap.get("dataSize") != null)
PreparedStatement statement = this.createPreparedStatement(this.getDataSizeInBytesSql);
if (statement == null)
{
// Number cast is necessary because the returned number can be an int or long
Number resultNumber = (Number) resultMap.get("dataSize");
long dataLength = resultNumber.longValue();
return dataLength;
return 0L;
}
else
try
{
return 0;
int i = 1;
statement.setInt(i++, detailLevel);
statement.setInt(i++, DhSectionPos.getX(pos));
statement.setInt(i++, DhSectionPos.getZ(pos));
ResultSet result = this.query(statement);
if (result == null || !result.next())
{
return 0L;
}
return result.getLong("dataSize");
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
}
private final String getTotalDataSizeInBytesSql =
"select SUM(LENGTH(Data)) as dataSize " +
"from "+this.getTableName()+"; ";
/** @return the total size in bytes of the full data for this entire database */
public long getTotalDataSizeInBytes()
{
Map<String, Object> resultMap = this.queryDictionaryFirst(
"select SUM(LENGTH(Data)) as dataSize " +
"from "+this.getTableName()+"; ");
PreparedStatement statement = this.createPreparedStatement(this.getTotalDataSizeInBytesSql);
if (resultMap != null && resultMap.get("dataSize") != null)
try
{
Number resultNumber = (Number) resultMap.get("dataSize");
long dataLength = resultNumber.longValue();
return dataLength;
ResultSet result = this.query(statement);
if (result == null || !result.next())
{
return 0;
}
return result.getLong("dataSize");
}
else
catch (SQLException e)
{
return 0;
throw new RuntimeException(e);
}
}
//===============//
// helper method //
//===============//
private static ByteArrayList putAllBytes(InputStream inputStream, @Nullable ByteArrayList existingArrayList) throws IOException
{
if (existingArrayList == null)
{
existingArrayList = new ByteArrayList(inputStream.available());
}
else
{
existingArrayList.clear();
existingArrayList.ensureCapacity(inputStream.available());
}
try
{
int nextByte = inputStream.read();
while (nextByte != -1)
{
existingArrayList.add((byte) nextByte);
nextByte = inputStream.read();
}
}
catch (EOFException ignore) { /* shouldn't happen, but just in case */ }
return existingArrayList;
}
}
core/src/main/java/com/seibel/distanthorizons/core/util/ListUtil.java
+40
View File
@@ -1,5 +1,9 @@
package com.seibel.distanthorizons.core.util;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import java.util.ArrayList;
public class ListUtil
@@ -16,4 +20,40 @@ public class ListUtil
}
/**
* Unlike {@link LongArrayList#ensureCapacity(int)} this method
* will populate the list with zeros up to the given size
* so get and set methods won't cause {@link IndexOutOfBoundsException}'s.
*/
public static void clearAndSetSize(LongArrayList arrayList, int size)
{
arrayList.clear();
for (int i = 0; i < size; i++)
{
arrayList.add(0L);
}
}
/** @see ListUtil#clearAndSetSize(LongArrayList, int) */
public static void clearAndSetSize(ShortArrayList arrayList, int size)
{
arrayList.clear();
for (int i = 0; i < size; i++)
{
arrayList.add((short)0);
}
}
/** @see ListUtil#clearAndSetSize(LongArrayList, int) */
public static void clearAndSetSize(ByteArrayList arrayList, int size)
{
arrayList.clear();
for (int i = 0; i < size; i++)
{
arrayList.add((byte)0);
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java
+53 -44
View File
@@ -22,8 +22,10 @@ package com.seibel.distanthorizons.core.util;
import com.google.common.annotations.VisibleForTesting;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException;
import it.unimi.dsi.fastutil.longs.LongArrays;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrays;
/**
@@ -43,7 +45,7 @@ import it.unimi.dsi.fastutil.shorts.ShortArrays;
*
* @author Builderb0y
*/
public class RenderDataPointReducingList
public class RenderDataPointReducingList extends PhantomArrayListParent
{
/**
@@ -97,7 +99,7 @@ public class RenderDataPointReducingList
*/
private short lowest, highest, smallest, biggest;
private short sizeWithAir, sizeWithoutAir;
private final long[] links, data;
private final LongArrayList links, data;
/**
* a temporary array to be used for sorting nodes.
* the array is first populated such that every index
@@ -106,7 +108,7 @@ public class RenderDataPointReducingList
* finally, the nodes are re-linked according
* to the order of elements in this array.
*/
private final short[] sortingArray;
private final ShortArrayList sortingArray;
@@ -116,6 +118,8 @@ public class RenderDataPointReducingList
public RenderDataPointReducingList(IColumnDataView view)
{
super(0, 1, 2);
int size = view.size();
if (size == 0)
{
@@ -123,11 +127,13 @@ public class RenderDataPointReducingList
this.setHighest(NULL);
this.setSmallest(NULL);
this.setBiggest(NULL);
this.links = LongArrays.EMPTY_ARRAY;
this.data = LongArrays.EMPTY_ARRAY;
this.sortingArray = ShortArrays.EMPTY_ARRAY;
this.links = this.pooledArraysCheckout.getLongArray(0);
this.data = this.pooledArraysCheckout.getLongArray(1);
this.sortingArray = this.pooledArraysCheckout.getShortArray(0);
if (ASSERTS) this.checkLinks();
this.pooledArraysCheckout = null;
return;
}
@@ -138,10 +144,13 @@ public class RenderDataPointReducingList
// We will use this array for sorting the nodes,
// first by lowest-to-highest, then by smallest-to-biggest.
int arrayCapacity = (size << 1) - 1;
this.sortingArray = new short[arrayCapacity];
this.links = new long[arrayCapacity];
java.util.Arrays.fill(this.links, DEFAULT_LINKS);
this.data = new long[arrayCapacity];
this.sortingArray = this.pooledArraysCheckout.getShortArray(0, arrayCapacity);
this.links = this.pooledArraysCheckout.getLongArray(0, arrayCapacity);
java.util.Arrays.fill(this.links.elements(), DEFAULT_LINKS);
this.data = this.pooledArraysCheckout.getLongArray(1, arrayCapacity);
this.pooledArraysCheckout = null;
int sizeWithoutAir = 0;
for (int index = 0; index < size; index++)
{
@@ -219,8 +228,8 @@ public class RenderDataPointReducingList
throw new IllegalArgumentException(RenderDataPointUtil.toString(lowerData) + " overlaps with " + RenderDataPointUtil.toString(higherData));
}
}
this.lowest = this.sortingArray[0];
this.highest = this.sortingArray[sizeWithoutAir - 1];
this.lowest = this.sortingArray.getShort(0);
this.highest = this.sortingArray.getShort(sizeWithoutAir - 1);
// now sort by size.
this.sortBySize(sizeWithAir);
@@ -232,8 +241,8 @@ public class RenderDataPointReducingList
this.setSmaller(biggerIndex, smallerIndex);
}
this.smallest = this.sortingArray[0];
this.biggest = this.sortingArray[sizeWithAir - 1];
this.smallest = this.sortingArray.getShort(0);
this.biggest = this.sortingArray.getShort(sizeWithAir - 1);
this.setSizeWithAir(sizeWithAir);
this.setSizeWithoutAir(sizeWithoutAir);
@@ -389,7 +398,7 @@ public class RenderDataPointReducingList
else this.setBiggest(smaller);
this.setData(index, DEFAUlT_DATA);
this.links[index] = DEFAULT_LINKS;
this.links.set(index, DEFAULT_LINKS);
this.sizeWithAir--;
if (isAlphaVisible(alpha)) this.sizeWithoutAir--;
@@ -415,11 +424,11 @@ public class RenderDataPointReducingList
}
long[] datas = this.data;
LongArrayList datas = this.data;
int writeIndex = 0;
for (int readIndex = this.getLowest(); readIndex != NULL; readIndex = this.getHigher(readIndex))
{
if (datas[readIndex] != DEFAUlT_DATA)
if (datas.getLong(readIndex) != DEFAUlT_DATA)
{
this.setSortingIndex(writeIndex++, readIndex);
}
@@ -434,8 +443,8 @@ public class RenderDataPointReducingList
this.setBigger(smaller, bigger);
}
this.smallest = this.sortingArray[0];
this.biggest = this.sortingArray[writeIndex - 1];
this.smallest = this.sortingArray.getShort(0);
this.biggest = this.sortingArray.getShort(writeIndex - 1);
this.setSmaller(this.getSmallest(), NULL);
this.setBigger(this.getBiggest(), NULL);
}
@@ -447,7 +456,7 @@ public class RenderDataPointReducingList
@VisibleForTesting
public void sortBySize(int size)
{
short[] array = this.sortingArray;
ShortArrayList array = this.sortingArray;
it.unimi.dsi.fastutil.Arrays.quickSort(
0,
size,
@@ -462,7 +471,7 @@ public class RenderDataPointReducingList
// swapper
(int index1, int index2) ->
{
ShortArrays.swap(array, index1, index2);
ShortArrays.swap(array.elements(), index1, index2);
}
);
}
@@ -474,7 +483,7 @@ public class RenderDataPointReducingList
@VisibleForTesting
public void sortByPosition(int size)
{
short[] array = this.sortingArray;
ShortArrayList array = this.sortingArray;
it.unimi.dsi.fastutil.Arrays.quickSort(
0,
size,
@@ -489,7 +498,7 @@ public class RenderDataPointReducingList
// swapper
(int index1, int index2) ->
{
ShortArrays.swap(array, index1, index2);
ShortArrays.swap(array.elements(), index1, index2);
}
);
}
@@ -913,15 +922,15 @@ public class RenderDataPointReducingList
public int getSizeWithAir() { return Short.toUnsignedInt(this.sizeWithAir); }
public int getSizeWithoutAir() { return Short.toUnsignedInt(this.sizeWithoutAir); }
public int getSortingIndex(int index) { return Short.toUnsignedInt(this.sortingArray[index]); }
public int getSortingIndex(int index) { return Short.toUnsignedInt(this.sortingArray.getShort(index)); }
public int getLower(int index) { return ((int) (this.links[index] >>> LOWER_SHIFT)) & LINK_MASK; }
public int getHigher(int index) { return ((int) (this.links[index] >>> HIGHER_SHIFT)) & LINK_MASK; }
public int getLower(int index) { return ((int) (this.links.getLong(index) >>> LOWER_SHIFT)) & LINK_MASK; }
public int getHigher(int index) { return ((int) (this.links.getLong(index) >>> HIGHER_SHIFT)) & LINK_MASK; }
public int getSmaller(int index) { return ((int) (this.links[index] >>> SMALLER_SHIFT)) & LINK_MASK; }
public int getBigger(int index) { return ((int) (this.links[index] >>> BIGGER_SHIFT)) & LINK_MASK; }
public int getSmaller(int index) { return ((int) (this.links.getLong(index) >>> SMALLER_SHIFT)) & LINK_MASK; }
public int getBigger(int index) { return ((int) (this.links.getLong(index) >>> BIGGER_SHIFT)) & LINK_MASK; }
public long getData(int index) { return this.data[index]; }
public long getData(int index) { return this.data.getLong(index); }
public int getMinY(int index) { return RenderDataPointUtil.getYMin(this.getData(index)); }
public int getMaxY(int index) { return RenderDataPointUtil.getYMax(this.getData(index)); }
@@ -955,62 +964,62 @@ public class RenderDataPointReducingList
public void setSizeWithAir(int sizeWithAir) { this.sizeWithAir = (short)(sizeWithAir); }
public void setSizeWithoutAir(int sizeWithoutAir) { this.sizeWithoutAir = (short)(sizeWithoutAir); }
public void setSortingIndex(int index, int to) { this.sortingArray[index] = (short)(to); }
public void setSortingIndex(int index, int to) { this.sortingArray.set(index, (short)to); }
public void setLower(int index, int lowerIndex)
{
this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << LOWER_SHIFT)) | (((long)(lowerIndex & LINK_MASK)) << LOWER_SHIFT);
this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << LOWER_SHIFT)) | (((long)(lowerIndex & LINK_MASK)) << LOWER_SHIFT));
}
public void setHigher(int index, int higherIndex)
{
this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << HIGHER_SHIFT)) | (((long)(higherIndex & LINK_MASK)) << HIGHER_SHIFT);
this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << HIGHER_SHIFT)) | (((long)(higherIndex & LINK_MASK)) << HIGHER_SHIFT));
}
public void setSmaller(int index, int smallerIndex)
{
this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << SMALLER_SHIFT)) | (((long)(smallerIndex & LINK_MASK)) << SMALLER_SHIFT);
this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << SMALLER_SHIFT)) | (((long)(smallerIndex & LINK_MASK)) << SMALLER_SHIFT));
}
public void setBigger(int index, int biggerIndex)
{
this.links[index] = (this.links[index] & ~(((long)(LINK_MASK)) << BIGGER_SHIFT)) | (((long)(biggerIndex & LINK_MASK)) << BIGGER_SHIFT);
this.links.set(index, (this.links.getLong(index) & ~(((long)(LINK_MASK)) << BIGGER_SHIFT)) | (((long)(biggerIndex & LINK_MASK)) << BIGGER_SHIFT));
}
public void setData(int index, long data) { this.data[index] = data; }
public void setData(int index, long data) { this.data.set(index, data); }
public void setMinY(int index, int minY)
{
this.data[index] = (this.data[index] & ~RenderDataPointUtil.DEPTH_SHIFTED_MASK) | ((minY & RenderDataPointUtil.DEPTH_MASK) << RenderDataPointUtil.DEPTH_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~RenderDataPointUtil.DEPTH_SHIFTED_MASK) | ((minY & RenderDataPointUtil.DEPTH_MASK) << RenderDataPointUtil.DEPTH_SHIFT));
}
public void setMaxY(int index, int maxY)
{
this.data[index] = (this.data[index] & ~RenderDataPointUtil.HEIGHT_SHIFTED_MASK) | ((maxY & RenderDataPointUtil.HEIGHT_MASK) << RenderDataPointUtil.HEIGHT_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~RenderDataPointUtil.HEIGHT_SHIFTED_MASK) | ((maxY & RenderDataPointUtil.HEIGHT_MASK) << RenderDataPointUtil.HEIGHT_SHIFT));
}
public void setRed(int index, int red)
{
this.data[index] = (this.data[index] & ~(RenderDataPointUtil.RED_MASK << RenderDataPointUtil.RED_SHIFT)) | ((red & RenderDataPointUtil.RED_MASK) << RenderDataPointUtil.RED_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.RED_MASK << RenderDataPointUtil.RED_SHIFT)) | ((red & RenderDataPointUtil.RED_MASK) << RenderDataPointUtil.RED_SHIFT));
}
public void setGreen(int index, int green)
{
this.data[index] = (this.data[index] & ~(RenderDataPointUtil.GREEN_MASK << RenderDataPointUtil.GREEN_SHIFT)) | ((green & RenderDataPointUtil.GREEN_MASK) << RenderDataPointUtil.GREEN_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.GREEN_MASK << RenderDataPointUtil.GREEN_SHIFT)) | ((green & RenderDataPointUtil.GREEN_MASK) << RenderDataPointUtil.GREEN_SHIFT));
}
public void setBlue(int index, int blue) {
this.data[index] = (this.data[index] & ~(RenderDataPointUtil.BLUE_MASK << RenderDataPointUtil.BLUE_SHIFT)) | ((blue & RenderDataPointUtil.BLUE_MASK) << RenderDataPointUtil.BLUE_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.BLUE_MASK << RenderDataPointUtil.BLUE_SHIFT)) | ((blue & RenderDataPointUtil.BLUE_MASK) << RenderDataPointUtil.BLUE_SHIFT));
}
public void setAlpha(int index, int alpha)
{
alpha >>>= RenderDataPointUtil.ALPHA_DOWNSIZE_SHIFT;
this.data[index] = (this.data[index] & ~(RenderDataPointUtil.ALPHA_MASK << RenderDataPointUtil.ALPHA_SHIFT)) | ((alpha & RenderDataPointUtil.ALPHA_MASK) << RenderDataPointUtil.ALPHA_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.ALPHA_MASK << RenderDataPointUtil.ALPHA_SHIFT)) | ((alpha & RenderDataPointUtil.ALPHA_MASK) << RenderDataPointUtil.ALPHA_SHIFT));
}
public void setBlockLight(int index, int blockLight)
{
this.data[index] = (this.data[index] & ~(RenderDataPointUtil.BLOCK_LIGHT_MASK << RenderDataPointUtil.BLOCK_LIGHT_SHIFT)) | ((blockLight & RenderDataPointUtil.BLOCK_LIGHT_MASK) << RenderDataPointUtil.BLOCK_LIGHT_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.BLOCK_LIGHT_MASK << RenderDataPointUtil.BLOCK_LIGHT_SHIFT)) | ((blockLight & RenderDataPointUtil.BLOCK_LIGHT_MASK) << RenderDataPointUtil.BLOCK_LIGHT_SHIFT));
}
public void setSkyLight(int index, int skyLight)
{
this.data[index] = (this.data[index] & ~(RenderDataPointUtil.SKY_LIGHT_MASK << RenderDataPointUtil.SKY_LIGHT_SHIFT)) | ((skyLight & RenderDataPointUtil.SKY_LIGHT_MASK) << RenderDataPointUtil.SKY_LIGHT_SHIFT);
this.data.set(index, (this.data.getLong(index) & ~(RenderDataPointUtil.SKY_LIGHT_MASK << RenderDataPointUtil.SKY_LIGHT_SHIFT)) | ((skyLight & RenderDataPointUtil.SKY_LIGHT_MASK) << RenderDataPointUtil.SKY_LIGHT_SHIFT));
}
core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java
+9 -3
View File
@@ -25,6 +25,8 @@ import com.seibel.distanthorizons.core.logging.SpamReducedLogger;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView;
import com.seibel.distanthorizons.coreapi.ModInfo;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/**
@@ -67,6 +69,8 @@ public class RenderDataPointUtil
public static final boolean RUN_VALIDATION = ModInfo.IS_DEV_BUILD;
private static final Logger LOGGER = LogManager.getLogger();
public final static int EMPTY_DATA = 0;
public final static int MAX_WORLD_Y_SIZE = 4096;
@@ -298,9 +302,11 @@ public class RenderDataPointUtil
}
else
{
RenderDataPointReducingList list = new RenderDataPointReducingList(sourceData);
list.reduce(output.verticalSize());
list.copyTo(output);
try (RenderDataPointReducingList list = new RenderDataPointReducingList(sourceData))
{
list.reduce(output.verticalSize());
list.copyTo(output);
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java
+25 -1
View File
@@ -26,6 +26,7 @@ import com.seibel.distanthorizons.coreapi.ModInfo;
import net.jpountz.lz4.LZ4FrameInputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.tukaani.xz.ResettableArrayCache;
import org.tukaani.xz.XZInputStream;
import java.io.*;
@@ -42,6 +43,8 @@ import java.io.*;
*/
public class DhDataInputStream extends DataInputStream
{
private static final ThreadLocal<ResettableArrayCache> LZMA_RESETTABLE_ARRAY_CACHE_GETTER = ThreadLocal.withInitial(() -> new ResettableArrayCache(new LzmaArrayCache()));
private static final Logger LOGGER = LogManager.getLogger();
@@ -60,8 +63,12 @@ public class DhDataInputStream extends DataInputStream
case LZ4:
return new LZ4FrameInputStream(stream);
case LZMA2:
// using an array cache significantly reduces GC pressure
ResettableArrayCache arrayCache = LZMA_RESETTABLE_ARRAY_CACHE_GETTER.get();
arrayCache.reset();
// Note: all LZMA/XZ compressors can be decompressed using this same InputStream
return new XZInputStream(stream);
return new XZInputStream(stream, arrayCache);
default:
throw new IllegalArgumentException("No compressor defined for [" + compressionMode + "]");
@@ -75,6 +82,23 @@ public class DhDataInputStream extends DataInputStream
}
}
@Override
public int read() throws IOException
{
try
{
return super.read();
}
catch (EOFException ignore)
{
// there's a bug with XZInputStream
// where it throws an EOFException instead
// of returning -1 as defined by DataInputStream.read()
return -1;
}
}
@Override
public void close() throws IOException { /* Do nothing. */ }
core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java
+1 -1
View File
@@ -53,7 +53,7 @@ public class LzmaArrayCache extends ArrayCache
public byte[] getByteArray(int size, boolean fillWithZeros)
{
ArrayList<byte[]> cacheList = this.byteCache.computeIfAbsent(size, (newSize) -> new ArrayList<>(4));
if (cacheList.size() == 0)
if (cacheList.isEmpty())
{
return new byte[size];
}
core/src/test/java/testItems/sql/TestCompoundKeyDto.java
+6
View File
@@ -61,4 +61,10 @@ public class TestCompoundKeyDto implements IBaseDTO<DhChunkPos>
return this.id + ", " + this.value;
}
@Override
public void close()
{ /* no closing needed */ }
}
core/src/test/java/testItems/sql/TestCompoundKeyRepo.java
+23 -9
View File
@@ -20,10 +20,13 @@
package testItems.sql;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;
@@ -45,22 +48,33 @@ public class TestCompoundKeyRepo extends AbstractDhRepo<DhChunkPos, TestCompound
"\n" +
",PRIMARY KEY (XPos, ZPos)" +
");";
this.queryDictionaryFirst(createTableSql);
PreparedStatement createTableStatement = this.createPreparedStatement(createTableSql);
this.query(createTableStatement);
}
@Override
public String getTableName() { return "TestCompound"; }
@Override
public String createWhereStatement(DhChunkPos key) { return "XPos = '"+key.getX()+"' AND ZPos = '"+key.getZ()+"'"; }
@Override
protected String CreateParameterizedWhereString() { return "XPos = ? AND ZPos = ?"; }
@Override
public TestCompoundKeyDto convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
@Override
protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, DhChunkPos pos) throws SQLException
{
int xPos = (int) objectMap.get("XPos");
int zPos = (int) objectMap.get("ZPos");
String value = (String) objectMap.get("Value");
statement.setInt(index++, pos.getX());
statement.setInt(index++, pos.getZ());
return index;
}
@Override
@Nullable
public TestCompoundKeyDto convertResultSetToDto(ResultSet result) throws ClassCastException, SQLException
{
int xPos = result.getInt("XPos");
int zPos = result.getInt("ZPos");
String value = result.getString("Value");
return new TestCompoundKeyDto(new DhChunkPos(xPos, zPos), value);
}
core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java
+22 -10
View File
@@ -19,10 +19,13 @@
package testItems.sql;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;
@@ -43,24 +46,33 @@ public class TestPrimaryKeyRepo extends AbstractDhRepo<Integer, TestSingleKeyDto
",LongValue BIGINT NULL\n" +
",ByteValue TINYINT NULL\n" +
");";
this.queryDictionaryFirst(createTableSql);
PreparedStatement createTableStatement = this.createPreparedStatement(createTableSql);
this.query(createTableStatement);
}
@Override
public String getTableName() { return "Test"; }
@Override
public String createWhereStatement(Integer keyString) { return "Id = '"+keyString+"'"; }
@Override
protected String CreateParameterizedWhereString() { return "Id = ?"; }
@Override
public TestSingleKeyDto convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
@Override
protected int setPreparedStatementWhereClause(PreparedStatement statement, int index, Integer id) throws SQLException
{
int id = (int) objectMap.get("Id");
String value = (String) objectMap.get("Value");
long longValue = (Long) objectMap.get("LongValue");
byte byteValue = (Byte) objectMap.get("ByteValue");
statement.setInt(index++, id);
return index;
}
@Override
@Nullable
public TestSingleKeyDto convertResultSetToDto(ResultSet result) throws ClassCastException, SQLException
{
int id = result.getInt("Id");
String value = result.getString("Value");
long longValue = result.getLong("LongValue");
byte byteValue = result.getByte("ByteValue");
return new TestSingleKeyDto(id, value, longValue, byteValue);
}
core/src/test/java/testItems/sql/TestSingleKeyDto.java
+6
View File
@@ -67,4 +67,10 @@ public class TestSingleKeyDto implements IBaseDTO<Integer>
return this.id + ", " + this.value + ", " + this.longValue + ", " + this.byteValue;
}
@Override
public void close()
{ /* no closing needed */ }
}
core/src/test/java/tests/DhFullDataSourceCacheTest.java
+220
View File
@@ -0,0 +1,220 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package tests;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.DatabaseUpdater;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import testItems.sql.TestCompoundKeyDto;
import testItems.sql.TestCompoundKeyRepo;
import testItems.sql.TestPrimaryKeyRepo;
import testItems.sql.TestSingleKeyDto;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Map;
/**
* Validates {@link AbstractDhRepo} is set up correctly.
*/
public class DhFullDataSourceCacheTest
{
public static String DATABASE_TYPE = "jdbc:sqlite";
public static String DB_FILE_NAME = "test.sqlite";
@BeforeClass
public static void testSetup()
{
File dbFile = new File(DB_FILE_NAME);
if (dbFile.exists())
{
Assert.assertTrue("unable to delete old test DB File.", dbFile.delete());
}
}
@Test
public void test()
{
FullDataSourceV2Repo repo = null;
try
{
repo = new FullDataSourceV2Repo(DATABASE_TYPE, new File(DB_FILE_NAME));
//========================//
// create test datasource //
//========================//
long pos = DhSectionPos.encode((byte)6, 1, 2);
FullDataPointIdMap dataMapping = new FullDataPointIdMap(pos);
LongArrayList[] fullDataArray = new LongArrayList[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++)
{
fullDataArray[i] = new LongArrayList(1);
for (int j = 0; j < 32; j++)
{
fullDataArray[i].add(FullDataPointUtil.encode(j, 1, j, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT));
}
}
byte[] columnGenStep = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
Arrays.fill(columnGenStep, (byte)3);
byte[] columnWorldCompressionMode = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
Arrays.fill(columnWorldCompressionMode, (byte)3);
FullDataSourceV2 originalDataSource = FullDataSourceV2.createWithData(pos, dataMapping, fullDataArray, columnGenStep, columnWorldCompressionMode);
FullDataSourceV2DTO originalDto = FullDataSourceV2DTO.CreateFromDataSource(originalDataSource, EDhApiDataCompressionMode.LZMA2);
repo.save(originalDto);
//=========================//
// assert DTO data is the same //
//=========================//
FullDataSourceV2DTO savedDto = repo.getByKey(pos);
Assert.assertNotNull("Failed to find DTO", savedDto);
Assert.assertEquals("Pos mismatch", originalDto.pos, savedDto.pos);
assertArraysAreEqual(originalDto.compressedDataByteArray, savedDto.compressedDataByteArray);
assertArraysAreEqual(originalDto.compressedColumnGenStepByteArray, savedDto.compressedColumnGenStepByteArray);
assertArraysAreEqual(originalDto.compressedWorldCompressionModeByteArray, savedDto.compressedWorldCompressionModeByteArray);
//====================================//
// assert dataSource data is the same //
//====================================//
FullDataSourceV2 savedDataSource = savedDto.createUnitTestDataSource();
Assert.assertNotNull("Failed to create DataSource", savedDataSource);
Assert.assertEquals("Pos mismatch", originalDataSource.getPos(), savedDataSource.getPos());
assertArraysAreEqual(originalDataSource.columnGenerationSteps, savedDataSource.columnGenerationSteps);
assertArraysAreEqual(originalDataSource.columnWorldCompressionMode, savedDataSource.columnWorldCompressionMode);
Assert.assertTrue(originalDataSource.dataPoints.length == savedDataSource.dataPoints.length);
for (int i = 0; i < FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH; i++)
{
assertArraysAreEqual(originalDataSource.dataPoints[i], savedDataSource.dataPoints[i]);
}
//=========================//
// (optional) loop forever //
//=========================//
// this can be set to "true"
// so we can profile the DTO creation process and
// see if there are any object leaks and how the GC
// handles it
if (false)
{
System.out.println("Initial save/get success, starting long update test for GC validation...");
for (int i = 0; i < 1_000_000; i++)
{
repo.save(originalDto);
try (FullDataSourceV2DTO pooledDto = repo.getByKey(pos))
{
Assert.assertNotNull(savedDto);
Assert.assertEquals(originalDto.pos, savedDto.pos);
assertArraysAreEqual(originalDto.compressedDataByteArray, savedDto.compressedDataByteArray);
assertArraysAreEqual(originalDto.compressedColumnGenStepByteArray, savedDto.compressedColumnGenStepByteArray);
assertArraysAreEqual(originalDto.compressedWorldCompressionModeByteArray, savedDto.compressedWorldCompressionModeByteArray);
}
}
}
}
catch (Exception e)
{
e.printStackTrace();
Assert.fail(e.getMessage());
}
finally
{
if (repo != null)
{
repo.close();
}
}
}
//================//
// helper methods //
//================//
private static void assertArraysAreEqual(ByteArrayList expectedArray, ByteArrayList actualArray)
{
Assert.assertEquals("size mismatch", expectedArray.size(), actualArray.size());
for (int i = 0; i < expectedArray.size(); i++)
{
byte expectedNumb = expectedArray.getByte(i);
byte actualNumb = actualArray.getByte(i);
Assert.assertEquals("value mismatch at index ["+i+"]", expectedNumb, actualNumb);
}
}
private static void assertArraysAreEqual(LongArrayList expectedArray, LongArrayList actualArray)
{
Assert.assertEquals("size mismatch", expectedArray.size(), actualArray.size());
for (int i = 0; i < expectedArray.size(); i++)
{
long expectedNumb = expectedArray.getLong(i);
long actualNumb = actualArray.getLong(i);
Assert.assertEquals("value mismatch at index ["+i+"]", expectedNumb, actualNumb);
}
}
}
core/src/test/java/tests/DhRepoSqliteTest.java
+5 -2
View File
@@ -31,6 +31,7 @@ import testItems.sql.TestPrimaryKeyRepo;
import testItems.sql.TestSingleKeyDto;
import java.io.File;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;
@@ -71,8 +72,10 @@ public class DhRepoSqliteTest
//==========================//
// check that the schema table is created
Map<String, Object> autoUpdateTablePresentResult = primaryKeyRepo.queryDictionaryFirst("SELECT name FROM sqlite_master WHERE type='table' AND name='"+DatabaseUpdater.SCHEMA_TABLE_NAME+"';");
if (autoUpdateTablePresentResult == null || autoUpdateTablePresentResult.get("name") == null)
ResultSet autoUpdateTablePresentResult = primaryKeyRepo.query(primaryKeyRepo.createPreparedStatement("SELECT name FROM sqlite_master WHERE type='table' AND name='"+DatabaseUpdater.SCHEMA_TABLE_NAME+"';"));
if (autoUpdateTablePresentResult == null
|| !autoUpdateTablePresentResult.next()
|| autoUpdateTablePresentResult.getString("name") == null)
{
Assert.fail("Auto DB update table missing.");
}