wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add FullDataSourceV2 pooling and replace long[][] arrays with LongArrayList[]

Browse Source
This commit is contained in:
James Seibel
2024-03-21 17:25:37 -05:00
parent 72c601b2db
commit f11e9a142f
12 changed files with 248 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java
Vendored
+4 -3
View File
@@ -46,6 +46,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import com.seibel.distanthorizons.coreapi.util.math.Vec3d;
import com.seibel.distanthorizons.coreapi.util.math.Vec3f;
import com.seibel.distanthorizons.coreapi.util.math.Vec3i;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -220,10 +221,10 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo
{
// attempt to get the LOD data from the data source
FullDataPointIdMap mapping = dataSource.mapping;
long[] dataColumn = dataSource.get(relativePos.x, relativePos.z);
LongArrayList dataColumn = dataSource.get(relativePos.x, relativePos.z);
if (dataColumn != null)
{
int dataColumnIndexCount = dataColumn.length;
int dataColumnIndexCount = dataColumn.size();
DhApiTerrainDataPoint[] returnArray = new DhApiTerrainDataPoint[dataColumnIndexCount];
long dataPoint;
@@ -234,7 +235,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo
// search for a datapoint that contains the block y position
for (int i = 0; i < dataColumnIndexCount; i++)
{
dataPoint = dataColumn[i];
dataPoint = dataColumn.getLong(i);
if (!getSpecificYCoordinate)
{
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java
+10
View File
@@ -387,6 +387,16 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
//==================//
// override methods //
//==================//
@Override
public void close() throws Exception
{ /* not currently needed */ }
//================//
// helper classes //
//================//
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+129 -101
View File
@@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
import com.seibel.distanthorizons.coreapi.ModInfo;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import javax.annotation.Nullable;
@@ -51,7 +52,7 @@ import java.util.concurrent.locks.ReentrantLock;
* @see FullDataPointUtilV2
* @see FullDataSourceV1
*/
public class FullDataSourceV2 implements IDataSource<IDhLevel>
public class FullDataSourceV2 implements IDataSource<IDhLevel>, AutoCloseable
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
/** useful for debugging, but can slow down update operations quite a bit due to being called so often. */
@@ -98,7 +99,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
* TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately
* there's something in the render data logic that expects this order so we can't change it right now
*/
public long[][] dataPoints;
public LongArrayList[] dataPoints;
public boolean isEmpty;
public boolean applyToParent = false;
@@ -113,7 +114,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
private FullDataSourceV2(DhSectionPos pos)
{
this.pos = pos;
this.dataPoints = new long[WIDTH * WIDTH][];
this.dataPoints = new LongArrayList[WIDTH * WIDTH];
this.mapping = new FullDataPointIdMap(pos);
this.isEmpty = true;
@@ -122,8 +123,8 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
this.columnGenerationSteps = new byte[WIDTH * WIDTH];
}
public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); }
private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps)
public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); }
private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps)
{
LodUtil.assertTrue(data.length == WIDTH * WIDTH);
@@ -150,7 +151,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// Note: this logic only works if the data point data is the same between both versions
byte[] columnGenerationSteps = new byte[WIDTH * WIDTH];
long[][] dataPoints = new long[WIDTH * WIDTH][];
LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH];
for (int x = 0; x < WIDTH; x++)
{
for (int z = 0; z < WIDTH; z++)
@@ -159,7 +160,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
if (dataColumn != null && dataColumn.length != 0)
{
int index = relativePosToIndex(x, z);
dataPoints[index] = dataColumn;
dataPoints[index] = new LongArrayList(dataColumn);
// convert the data point format
@@ -212,13 +213,13 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
long[] legacyDataColumn = legacyData.get(x, z);
if (legacyDataColumn != null && legacyDataColumn.length != 0)
{
long[] newDataColumn = fullDataSource.get(x, z);
LongArrayList newDataColumn = fullDataSource.get(x, z);
if (newDataColumn == null)
{
LodUtil.assertNotReach("Accessor column mismatch");
}
else if (legacyDataColumn.length != newDataColumn.length)
else if (legacyDataColumn.length != newDataColumn.size())
{
LodUtil.assertNotReach("Accessor column length mismatch");
}
@@ -226,7 +227,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
{
for (int i = 0; i < legacyDataColumn.length; i++)
{
if (legacyDataColumn[i] != newDataColumn[i])
if (legacyDataColumn[i] != newDataColumn.getLong(i))
{
LodUtil.assertNotReach("Data mismatch");
}
@@ -247,7 +248,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// data //
//======//
public long[] get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; }
public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; }
@Override
public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); }
@@ -309,7 +310,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
{
int index = relativePosToIndex(x, z);
long[] newDataArray = inputDataSource.dataPoints[index];
LongArrayList newDataArray = inputDataSource.dataPoints[index];
if (newDataArray != null)
{
byte thisGenState = this.columnGenerationSteps[index];
@@ -318,11 +319,15 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value
&& thisGenState <= inputGenState)
{
long[] oldDataArray = this.dataPoints[index];
LongArrayList oldDataArray = this.dataPoints[index];
// copy over the new data
this.dataPoints[index] = new long[newDataArray.length];
System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length);
if (this.dataPoints[index] == null)
{
this.dataPoints[index] = new LongArrayList(new long[newDataArray.size()]);
}
this.dataPoints[index].clear();
this.dataPoints[index].addAll(newDataArray);
this.remapDataColumn(index, remappedIds);
if (RUN_DATA_ORDER_VALIDATION)
@@ -383,8 +388,8 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
this.columnGenerationSteps[recipientIndex] = inputGenStep;
// data points
long[] oldDataArray = this.dataPoints[recipientIndex];
long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z);
LongArrayList oldDataArray = this.dataPoints[recipientIndex];
LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z);
this.dataPoints[recipientIndex] = mergedInputDataArray;
if (RUN_DATA_ORDER_VALIDATION)
@@ -427,9 +432,13 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
}
return minWorldGenStepValue;
}
private static long[] mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z)
private static ThreadLocal<LongArrayList> mergeTwoByTwoTempList = ThreadLocal.withInitial(() -> new LongArrayList(20));
private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z)
{
ArrayList<Long> newColumnList = new ArrayList<>();
LongArrayList newColumnList = mergeTwoByTwoTempList.get();
newColumnList.clear();
// special numbers:
// -2 = the column's height hasn't been determined yet
@@ -466,8 +475,8 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++)
{
// TODO throw an assertion if the column isn't in top-down order or just fix it...
long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)];
if (inputDataArray == null || inputDataArray.length == 0)
LongArrayList inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)];
if (inputDataArray == null || inputDataArray.size() == 0)
{
currentDatapointIndex[colIndex] = -1;
continue;
@@ -476,7 +485,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// determine the last index (the lowest data point) for each column
if (currentDatapointIndex[colIndex] == -2)
{
currentDatapointIndex[colIndex] = inputDataArray.length - 1;
currentDatapointIndex[colIndex] = inputDataArray.size() - 1;
if (RUN_DATA_ORDER_VALIDATION)
{
@@ -491,7 +500,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// went over the end
continue;
}
long datapoint = inputDataArray[dataPointIndex];
long datapoint = inputDataArray.getLong(dataPointIndex);
int datapointMinY = FullDataPointUtilV2.getBottomY(datapoint);
int numbOfBlocksTall = FullDataPointUtilV2.getHeight(datapoint);
@@ -575,10 +584,10 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// convert the arraylist to an array
long[]mergedInputDataArray = new long[newColumnList.size()];
for (int i = 0; i < mergedInputDataArray.length; i++)
LongArrayList mergedInputDataArray = new LongArrayList(new long[newColumnList.size()]);
for (int i = 0; i < mergedInputDataArray.size(); i++)
{
mergedInputDataArray[i] = newColumnList.get(i);
mergedInputDataArray.set(i, newColumnList.getLong(i));
}
@@ -586,10 +595,10 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// TODO why is this sometimes necessary? What did I (James) screw up that causes the mergedInputDataArray
// to sometimes be in a different order? Is it potentially related to what detail level is coming in?
{
long firstDataPoint = mergedInputDataArray[0];
long firstDataPoint = mergedInputDataArray.getLong(0);
int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint);
long lastDataPoint = mergedInputDataArray[mergedInputDataArray.length - 1];
long lastDataPoint = mergedInputDataArray.getLong(mergedInputDataArray.size() - 1);
int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint);
if (firstBottomY < lastBottomY)
@@ -597,11 +606,11 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// reverse the array so index 0 is the highest,
// this is necessary for later logic
// source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java
for(int i = 0; i < mergedInputDataArray.length / 2; i++)
for(int i = 0; i < mergedInputDataArray.size() / 2; i++)
{
long temp = mergedInputDataArray[i];
mergedInputDataArray[i] = mergedInputDataArray[mergedInputDataArray.length - i - 1];
mergedInputDataArray[mergedInputDataArray.length - i - 1] = temp;
long temp = mergedInputDataArray.getLong(i);
mergedInputDataArray.set(i, mergedInputDataArray.getLong(mergedInputDataArray.size() - i - 1));
mergedInputDataArray.set(mergedInputDataArray.size() - i - 1, temp);
}
}
}
@@ -615,15 +624,15 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
*/
private void remapDataColumn(int dataPointIndex, int[] remappedIds)
{
long[] dataColumn = this.dataPoints[dataPointIndex];
for (int i = 0; i < dataColumn.length; i++)
LongArrayList dataColumn = this.dataPoints[dataPointIndex];
for (int i = 0; i < dataColumn.size(); i++)
{
dataColumn[i] = FullDataPointUtilV2.remap(remappedIds, dataColumn[i]);
dataColumn.set(i, FullDataPointUtilV2.remap(remappedIds, dataColumn.getLong(i)));
}
}
private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray)
private static boolean areDataColumnsDifferent(LongArrayList oldDataArray, LongArrayList newDataArray)
{
if (oldDataArray == null || oldDataArray.length != newDataArray.length)
if (oldDataArray == null || oldDataArray.size() != newDataArray.size())
{
// new data was added/removed
return true;
@@ -631,8 +640,8 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
else
{
// check if the new column data is different
int oldArrayHash = Arrays.hashCode(oldDataArray);
int newArrayHash = Arrays.hashCode(newDataArray);
int oldArrayHash = oldDataArray.hashCode();
int newArrayHash = newDataArray.hashCode();
return (newArrayHash != oldArrayHash);
}
}
@@ -732,12 +741,12 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
*
* @see FullDataSourceV2#dataPoints
*/
public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, long[] dataArray) throws IllegalStateException
public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, LongArrayList dataArray) throws IllegalStateException
{
long firstDataPoint = dataArray[0];
long firstDataPoint = dataArray.getLong(0);
int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint);
long lastDataPoint = dataArray[dataArray.length - 1];
long lastDataPoint = dataArray.getLong(dataArray.size() - 1);
int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint);
if (firstBottomY < lastBottomY)
@@ -767,7 +776,7 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]);
}
public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep)
public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep)
{
int index = relativePosToIndex(relX, relZ);
this.dataPoints[index] = longArray;
@@ -778,9 +787,9 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
{
// validate the incoming ID's
int maxValidId = this.mapping.getMaxValidId();
for (int i = 0; i < longArray.length; i++)
for (int i = 0; i < longArray.size(); i++)
{
long dataPoint = longArray[i];
long dataPoint = longArray.getLong(i);
int id = FullDataPointUtilV2.getId(dataPoint);
if (id > maxValidId)
{
@@ -858,64 +867,83 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
// pooling //
//=========//
// TODO add pooled data sources
private static class Pooling
@Override
public void close() throws Exception
{
/** used when pooling data sources */
private final ArrayList<FullDataSourceV1> cachedSources = new ArrayList<>();
private final ReentrantLock cacheLock = new ReentrantLock();
/** @return null if no pooled source exists */
public FullDataSourceV1 tryGetPooledSource()
{
try
{
this.cacheLock.lock();
int index = this.cachedSources.size() - 1;
if (index == -1)
{
return null;
}
else
{
return this.cachedSources.remove(index);
}
}
finally
{
this.cacheLock.unlock();
}
}
/**
* Doesn't have to be called, if a data source isn't returned, nothing will be leaked.
* It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called.
*/
public void returnPooledDataSource(FullDataSourceV1 dataSource)
{
if (dataSource == null)
{
return;
}
else if (this.cachedSources.size() > 25)
{
return;
}
try
{
this.cacheLock.lock();
this.cachedSources.add(dataSource);
}
finally
{
this.cacheLock.unlock();
}
}
returnPooledDataSource(this);
}
/** used when pooling data sources */
private static final ArrayList<FullDataSourceV2> CACHED_SOURCES = new ArrayList<>();
private static final ReentrantLock CACHE_LOCK = new ReentrantLock();
/** @return an empty data source if non are cached */
public static FullDataSourceV2 getPooledSource(DhSectionPos pos, boolean clearData)
{
try
{
CACHE_LOCK.lock();
int index = CACHED_SOURCES.size() - 1;
if (index == -1)
{
return createEmpty(pos);
}
else
{
FullDataSourceV2 dataSource = CACHED_SOURCES.remove(index);;
dataSource.pos = pos;
if (clearData)
{
dataSource.mapping.clear(pos);
for (int i = 0; i < dataSource.dataPoints.length; i++)
{
if (dataSource.dataPoints[i] != null)
{
dataSource.dataPoints[i].clear();
}
}
Arrays.fill(dataSource.columnGenerationSteps, (byte) 0);
}
return dataSource;
}
}
finally
{
CACHE_LOCK.unlock();
}
}
/**
* Doesn't have to be called, if a data source isn't returned, nothing will be leaked.
* It just means a new source must be constructed next time {@link FullDataSourceV2#getPooledSource} is called.
*/
public static void returnPooledDataSource(FullDataSourceV2 dataSource)
{
if (dataSource == null)
{
return;
}
else if (CACHED_SOURCES.size() > 25)
{
return;
}
try
{
CACHE_LOCK.lock();
CACHED_SOURCES.add(dataSource);
}
finally
{
CACHE_LOCK.unlock();
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
+6 -1
View File
@@ -37,6 +37,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import com.seibel.distanthorizons.core.util.ColorUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import java.io.*;
@@ -317,7 +318,7 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z);
int columnHash = columnArrayView.getDataHash();
long[] dataColumn = inputFullDataSource.get(x, z);
LongArrayList dataColumn = inputFullDataSource.get(x, z);
EDhApiWorldGenerationStep worldGenStep = inputFullDataSource.getWorldGenStepAtRelativePos(x, z);
if (dataColumn != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY)
{
@@ -476,6 +477,10 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
return stringBuilder.toString();
}
@Override
public void close() throws Exception
{ /* not currently needed */ }
//==============//
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java
+8 -7
View File
@@ -36,6 +36,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import java.util.HashSet;
@@ -117,7 +118,7 @@ public class FullDataToRenderDataTransformer
throwIfThreadInterrupted();
ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z);
long[] dataColumn = fullDataSource.get(x, z);
LongArrayList dataColumn = fullDataSource.get(x, z);
convertColumnData(level, fullDataSource.mapping, baseX + x, baseZ + z, columnArrayView, dataColumn);
}
}
@@ -159,7 +160,7 @@ public class FullDataToRenderDataTransformer
private static void iterateAndConvert(
IDhClientLevel level, FullDataPointIdMap fullDataMapping,
int blockX, int blockZ,
ColumnArrayView renderColumnData, long[] fullColumnData)
ColumnArrayView renderColumnData, LongArrayList fullColumnData)
{
boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING);
boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get();
@@ -171,9 +172,9 @@ public class FullDataToRenderDataTransformer
int columnOffset = 0;
// goes from the top down
for (int i = 0; i < fullColumnData.length; i++)
for (int i = 0; i < fullColumnData.size(); i++)
{
long fullData = fullColumnData[i];
long fullData = fullColumnData.getLong(i);
int bottomY = FullDataPointUtilV2.getBottomY(fullData);
int blockHeight = FullDataPointUtilV2.getHeight(fullData);
int id = FullDataPointUtilV2.getId(fullData);
@@ -270,14 +271,14 @@ public class FullDataToRenderDataTransformer
}
// TODO what does this mean?
public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, long[] fullDataColumn)
public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, LongArrayList fullDataColumn)
{
if (fullDataColumn == null || fullDataColumn.length == 0)
if (fullDataColumn == null || fullDataColumn.size() == 0)
{
return;
}
int dataTotalLength = fullDataColumn.length;
int dataTotalLength = fullDataColumn.size();
if (dataTotalLength > columnArrayView.verticalSize())
{
ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength);
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java
+6 -5
View File
@@ -131,7 +131,7 @@ public class LodDataBuilder
{
for (int chunkZ = 0; chunkZ < LodUtil.CHUNK_WIDTH; chunkZ++)
{
LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4);
LongArrayList longs = new LongArrayList(new long[chunkWrapper.getHeight() / 4]);
int lastY = chunkWrapper.getMaxBuildHeight();
IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ);
IBlockStateWrapper blockState = AIR;
@@ -201,7 +201,7 @@ public class LodDataBuilder
}
longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight));
dataSource.setSingleColumn(longs.toLongArray(),
dataSource.setSingleColumn(longs,
chunkX + chunkOffsetX,
chunkZ + chunkOffsetZ,
EDhApiWorldGenerationStep.LIGHT);
@@ -230,7 +230,7 @@ public class LodDataBuilder
// AND the below loop won't run.
int size = (columnDataPoints != null) ? columnDataPoints.size() : 0;
long[] packedDataPoints = new long[size];
LongArrayList packedDataPoints = new LongArrayList(new long[size]);
for (int index = 0; index < size; index++)
{
DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index);
@@ -240,13 +240,14 @@ public class LodDataBuilder
(IBlockStateWrapper) (dataPoint.blockStateWrapper)
);
packedDataPoints[index] = FullDataPointUtilV2.encode(
packedDataPoints.set(index,
FullDataPointUtilV2.encode(
id,
dataPoint.topYBlockPos - dataPoint.bottomYBlockPos,
dataPoint.bottomYBlockPos - dataPoints.topYBlockPos,
(byte) (dataPoint.blockLightLevel),
(byte) (dataPoint.skyLightLevel)
);
));
}
accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT);
core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java
+14 -13
View File
@@ -221,22 +221,23 @@ public abstract class AbstractNewDataSourceHandler
}
// get or create the data source
TDataSource recipientDataSource = this.get(updatePos);
boolean dataModified = recipientDataSource.update(inputData, this.level);
if (dataModified)
// try block allows for disposing of pooled data sources after the update is complete
try (TDataSource recipientDataSource = this.get(updatePos))
{
// save the updated data to the database
TDTO dto = this.createDtoFromDataSource(recipientDataSource);
this.repo.save(dto);
for (IDataSourceUpdateFunc<TDataSource> listener : this.dateSourceUpdateListeners)
boolean dataModified = recipientDataSource.update(inputData, this.level);
if (dataModified)
{
if (listener != null)
// save the updated data to the database
TDTO dto = this.createDtoFromDataSource(recipientDataSource);
this.repo.save(dto);
for (IDataSourceUpdateFunc<TDataSource> listener : this.dateSourceUpdateListeners)
{
listener.OnDataSourceUpdated(recipientDataSource);
if (listener != null)
{
listener.OnDataSourceUpdated(recipientDataSource);
}
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java
+4 -2
View File
@@ -11,11 +11,13 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre
import java.io.IOException;
/**
* Base for all data sources.
* Base for all data sources. <br><br>
*
* AutoCloseable Can be implemented to allow for disposing of pooled data sources. <br><br>
*
* @param <TDhLevel> there are times when we need specifically a client level vs a more generic level
*/
public interface IDataSource<TDhLevel extends IDhLevel> extends IBaseDTO<DhSectionPos>
public interface IDataSource<TDhLevel extends IDhLevel> extends IBaseDTO<DhSectionPos>, AutoCloseable
{
DhSectionPos getSectionPos();
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java
+7 -5
View File
@@ -162,11 +162,11 @@ public class FullDataSourceProviderV2
{
// TODO maybe just set children update flags to true?
// TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway
return FullDataSourceV2.createEmpty(pos);
return FullDataSourceV2.getPooledSource(pos, true);
}
@Override
protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.createEmpty(pos); }
protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.getPooledSource(pos, true); }
@@ -248,9 +248,11 @@ public class FullDataSourceProviderV2
childReadLock.lock();
this.lockedPosSet.add(childPos);
FullDataSourceV2 dataSource = this.get(childPos);
this.updateDataSourceAtPos(parentUpdatePos, dataSource, false);
this.repo.setApplyToParent(childPos, false);
try (FullDataSourceV2 dataSource = this.get(childPos))
{
this.updateDataSourceAtPos(parentUpdatePos, dataSource, false);
this.repo.setApplyToParent(childPos, false);
}
}
catch (Exception e)
{
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java
+6 -2
View File
@@ -92,8 +92,12 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler<Col
{
ColumnRenderSource renderDataSource;
FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(pos);
renderDataSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level);
try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(pos))
{
renderDataSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level);
}
catch (Exception e) { throw new RuntimeException(e); }
return renderDataSource;
}
core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java
+7 -6
View File
@@ -41,6 +41,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftCli
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.LogManager;
import java.io.File;
@@ -233,8 +234,8 @@ public class SubDimensionLevelMatcher implements AutoCloseable
{
for (int z = 0; z < FullDataSourceV1.WIDTH; z++)
{
long[] newColumn = newDataSource.get(x, z);
long[] testColumn = testFullDataSource.get(x, z);
LongArrayList newColumn = newDataSource.get(x, z);
LongArrayList testColumn = testFullDataSource.get(x, z);
if (newColumn != null && testColumn != null)
{
@@ -244,11 +245,11 @@ public class SubDimensionLevelMatcher implements AutoCloseable
FullDataPointIdMap testDataMap = testFullDataSource.mapping;
// use min to prevent going out of bounds
int minColumnIndex = Math.min(newColumn.length, testColumn.length);
int minColumnIndex = Math.min(newColumn.size(), testColumn.size());
for (int i = 0; i < minColumnIndex; i++)
{
long newDataPoint = newColumn[i];
long testDataPoint = testColumn[i];
long newDataPoint = newColumn.getLong(i);
long testDataPoint = testColumn.getLong(i);
int newId = FullDataPointUtilV2.getId(newDataPoint);
int testId = FullDataPointUtilV2.getId(testDataPoint);
@@ -297,7 +298,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable
else if (newColumn != null)
{
// missing test column
totalDataPointCount += newColumn.length;
totalDataPointCount += newColumn.size();
}
else
{
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+47 -15
View File
@@ -23,10 +23,14 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongArrays;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import java.io.ByteArrayInputStream;
@@ -38,6 +42,9 @@ import java.util.zip.CheckedOutputStream;
/** handles storing {@link FullDataSourceV2}'s in the database. */
public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
public DhSectionPos pos;
public int levelMinY;
@@ -113,7 +120,10 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
//========================//
public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException
{ return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), levelWrapper); }
{
FullDataSourceV2 dataSource = FullDataSourceV2.getPooledSource(this.pos, false);
return this.populateDataSource(dataSource, levelWrapper);
}
public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException
{ return this.internalPopulateDataSource(dataSource, levelWrapper, false); }
@@ -133,7 +143,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
}
dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum);
dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum);
dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, this.compressionModeEnum);
dataSource.mapping.clear(dataSource.getSectionPos());
// should only be null when used in a unit test
@@ -163,7 +173,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
// (de)serializing //
//=================//
private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
private static CheckedByteArray writeDataSourceDataArrayToBlob(LongArrayList[] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
// write the outputs to a stream to prep for writing to the database
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -179,10 +189,10 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH;
for (int xz = 0; xz < dataArrayLength; xz++)
{
long[] dataColumn = dataArray[xz];
LongArrayList dataColumn = dataArray[xz];
// write column length
short columnLength = (dataColumn != null) ? (short) dataColumn.length : 0;
short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0;
// a short is used instead of an int because at most we store 4096 vertical slices and a
// short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion)
compressedOut.writeShort(columnLength);
@@ -190,7 +200,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
// write column data (will be skipped if no data was present)
for (int y = 0; y < columnLength; y++)
{
compressedOut.writeLong(dataColumn[y]);
compressedOut.writeLong(dataColumn.getLong(y));
}
}
@@ -202,33 +212,55 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray());
}
private static long[][] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, LongArrayList[] existingDataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray);
DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum);
int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH;
if (existingDataArray == null
|| existingDataArray.length != dataArrayLength)
{
existingDataArray = new LongArrayList[dataArrayLength];
}
// read the data
int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH;
long[][] dataArray = new long[dataArrayLength][];
for (int xz = 0; xz < dataArray.length; xz++)
for (int xz = 0; xz < existingDataArray.length; xz++)
{
// read the column length
short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later
long[] dataColumn = new long[dataColumnLength];
short dataColumnLength = compressedIn.readShort();
// use the existing array if possible
LongArrayList dataColumn = existingDataArray[xz];
if (dataColumn == null)
{
dataColumn = new LongArrayList(new long[dataColumnLength]);
existingDataArray[xz] = dataColumn;
}
dataColumn.clear();
dataColumn.ensureCapacity(dataColumnLength);
while (dataColumn.size() < dataColumnLength)
{
dataColumn.add(0);
}
// read column data (will be skipped if no data was present)
for (int y = 0; y < dataColumnLength; y++)
{
long dataPoint = compressedIn.readLong();
dataColumn[y] = dataPoint;
dataColumn.set(y, dataPoint);
}
dataArray[xz] = dataColumn;
existingDataArray[xz] = dataColumn;
}
return dataArray;
return existingDataArray;
}