pool columnRenderSources

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java

@@ -24,8 +24,8 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio
 import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
 import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder;
 import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler;
+import com.seibel.distanthorizons.core.file.DataSourcePool;
 import com.seibel.distanthorizons.core.file.IDataSource;
-import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
@@ -33,7 +33,6 @@ import com.seibel.distanthorizons.core.util.FullDataPointUtilV2;
 import com.seibel.distanthorizons.core.util.FullDataPointUtilV1;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
-import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
 import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
 import com.seibel.distanthorizons.coreapi.ModInfo;
@@ -41,16 +40,11 @@ import it.unimi.dsi.fastutil.longs.LongArrayList;
 import org.apache.logging.log4j.Logger;
 
 import javax.annotation.Nullable;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * This data source contains every datapoint over its given {@link DhSectionPos}. <br><br>
  * 
- * TODO create a child object that extends AutoClosable
- *       that can be pooled to reduce GC overhead 
- * 
  * @see FullDataPointUtilV2
  * @see FullDataSourceV1
  */
@@ -71,7 +65,9 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
 	public static final int WIDTH = 64;
 	
 	public static final byte DATA_FORMAT_VERSION = 1;
-		
+	
+	public static final DataSourcePool<FullDataSourceV2, IDhLevel> DATA_SOURCE_POOL = new DataSourcePool<>(FullDataSourceV2::createEmpty, FullDataSourceV2::prepPooledDataSource);
+	
 	
 	
 	private int cachedHashCode = 0;
@@ -836,6 +832,33 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
 	
 	
 	
+	//=========//
+	// pooling //
+	//=========//
+	
+	private static void prepPooledDataSource(DhSectionPos pos, boolean clearData, FullDataSourceV2 dataSource)
+	{
+		dataSource.pos = pos;
+		
+		if (clearData)
+		{
+			dataSource.mapping.clear(pos);
+			
+			for (int i = 0; i < dataSource.dataPoints.length; i++)
+			{
+				if (dataSource.dataPoints[i] != null)
+				{
+					dataSource.dataPoints[i].clear();
+				}
+			}
+			
+			Arrays.fill(dataSource.columnGenerationSteps, (byte) 0);
+			Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0);
+		}
+	}
+	
+	
+	
 	//=====================//
 	// setters and getters //
 	//=====================//
@@ -929,92 +952,10 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
 		}
 	}
 	
-	
-	
-	//=========//
-	// pooling //
-	//=========//
-	
 	@Override
 	public void close() throws Exception
 	{
-		returnPooledDataSource(this);
+		DATA_SOURCE_POOL.returnPooledDataSource(this);
 	}
 	
-	
-	/** used when pooling data sources */
-	private static final ArrayList<FullDataSourceV2> CACHED_SOURCES = new ArrayList<>();
-	private static final ReentrantLock CACHE_LOCK = new ReentrantLock();
-	
-	
-	/** @return an empty data source if non are cached */
-	public static FullDataSourceV2 getPooledSource(DhSectionPos pos, boolean clearData)
-	{
-		try
-		{
-			CACHE_LOCK.lock();
-			
-			int index = CACHED_SOURCES.size() - 1;
-			if (index == -1)
-			{
-				// no pooled sources exist
-				return createEmpty(pos);
-			}
-			else
-			{
-				FullDataSourceV2 dataSource = CACHED_SOURCES.remove(index);
-				dataSource.pos = pos;
-				
-				if (clearData)
-				{
-					dataSource.mapping.clear(pos);
-					
-					for (int i = 0; i < dataSource.dataPoints.length; i++)
-					{
-						if (dataSource.dataPoints[i] != null)
-						{
-							dataSource.dataPoints[i].clear();
-						}
-					}
-					
-					Arrays.fill(dataSource.columnGenerationSteps, (byte) 0);
-					Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0);
-				}
-				
-				return dataSource;
-			}
-		}
-		finally
-		{
-			CACHE_LOCK.unlock();
-		}
-	}
-	
-	/**
-	 * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. 
-	 * It just means a new source must be constructed next time {@link FullDataSourceV2#getPooledSource} is called.
-	 */
-	public static void returnPooledDataSource(FullDataSourceV2 dataSource)
-	{
-		if (dataSource == null)
-		{
-			return;
-		}
-		else if (CACHED_SOURCES.size() > 25)
-		{
-			return;
-		}
-		
-		try
-		{
-			CACHE_LOCK.lock();
-			CACHED_SOURCES.add(dataSource);
-		}
-		finally
-		{
-			CACHE_LOCK.unlock();
-		}
-	}
-	
-	
 }

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java

@@ -22,25 +22,22 @@ package com.seibel.distanthorizons.core.dataObjects.render;
 import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
+import com.seibel.distanthorizons.core.file.DataSourcePool;
 import com.seibel.distanthorizons.core.file.IDataSource;
-import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
-import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
 import com.seibel.distanthorizons.coreapi.ModInfo;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnQuadView;
-import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView;
 import com.seibel.distanthorizons.core.level.IDhClientLevel;
 import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
 import com.seibel.distanthorizons.core.util.ColorUtil;
 import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
-import com.seibel.distanthorizons.core.util.LodUtil;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import org.apache.logging.log4j.Logger;
 
-import java.io.*;
+import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
@@ -56,27 +53,21 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
 	public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
 	public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
 	
+	@Deprecated
 	public static final byte DATA_FORMAT_VERSION = 1;
 	@Override
 	public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; }
 	
-	public static final String DATA_NAME = "ColumnRenderSource";
+	public static final DataSourcePool<ColumnRenderSource, IDhClientLevel> DATA_SOURCE_POOL = new DataSourcePool<>(ColumnRenderSource::createEmptyRenderSource, null /* data source prep/cleanup needs to be done outside the pool since it requires additional inputs */);
 	
-	/**
-	 * This is the byte put between different sections in the binary save file.
-	 * The presence and absence of this byte indicates if the file is correctly formatted.
-	 */
-	public static final int DATA_GUARD_BYTE = 0xFFFFFFFF;
-	/** indicates the binary save file represents an empty data source */
-	public static final int NO_DATA_FLAG_BYTE = 0x00000001;
 	
 	
 	/** will be zero if an empty data source was created */
 	public int verticalDataCount;
-	public final DhSectionPos sectionPos;
-	public final int yOffset;
+	public DhSectionPos sectionPos;
+	public int yOffset;
 	
-	public long[] renderDataContainer;
+	public LongArrayList renderDataContainer;
 	
 	public final DebugSourceFlag[] debugSourceFlags;
 	
@@ -91,19 +82,52 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
 	// constructors //
 	//==============//
 	
-	public static ColumnRenderSource createEmptyRenderSource(DhSectionPos sectionPos) { return new ColumnRenderSource(sectionPos, 0, 0); }
+	/** 
+	 * This is separate from {@link DataSourcePool#getPooledSource(DhSectionPos, boolean)} 
+	 * because we need to pass in a couple extra values, 
+	 * specifically maxVerticalSize and yOffset.
+	 */
+	public static ColumnRenderSource getPooledRenderSource(DhSectionPos pos, int maxVerticalSize, int yOffset, boolean clearData)
+	{
+		ColumnRenderSource renderSource = DATA_SOURCE_POOL.getPooledSource(pos);
+		
+		// set necessary properties
+		renderSource.sectionPos = pos;
+		renderSource.verticalDataCount = maxVerticalSize;
+		renderSource.yOffset = yOffset;
+		
+		
+		// resize the array if necessary
+		int dataArraySize = SECTION_SIZE * SECTION_SIZE * maxVerticalSize;
+		renderSource.renderDataContainer.ensureCapacity(dataArraySize);
+		while (renderSource.renderDataContainer.size() < dataArraySize)
+		{
+			renderSource.renderDataContainer.add(0);
+		}
+		
+		if (clearData)
+		{
+			Arrays.fill(renderSource.renderDataContainer.elements(), 0);
+			Arrays.fill(renderSource.debugSourceFlags, null);
+		}
+		
+		return renderSource;
+	}
+	
+	
+	private static ColumnRenderSource createEmptyRenderSource(DhSectionPos sectionPos) { return new ColumnRenderSource(sectionPos, 0, 0); }
 	/**
 	 * Creates an empty ColumnRenderSource.
 	 *
-	 * @param sectionPos the relative position of the container
+	 * @param pos the relative position of the container
 	 * @param maxVerticalSize the maximum vertical size of the container
 	 */
-	public ColumnRenderSource(DhSectionPos sectionPos, int maxVerticalSize, int yOffset)
+	private ColumnRenderSource(DhSectionPos pos, int maxVerticalSize, int yOffset)
 	{
 		this.verticalDataCount = maxVerticalSize;
-		this.renderDataContainer = new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount];
+		this.renderDataContainer = new LongArrayList(new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]);
 		this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE];
-		this.sectionPos = sectionPos;
+		this.sectionPos = pos;
 		this.yOffset = yOffset;
 		this.worldGenStep = EDhApiWorldGenerationStep.EMPTY;
 	}
@@ -114,65 +138,7 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
 	// datapoint manipulation //
 	//========================//
 	
-	public void clearDataPoint(int posX, int posZ)
-	{
-		for (int verticalIndex = 0; verticalIndex < this.verticalDataCount; verticalIndex++)
-		{
-			this.renderDataContainer[posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex] = RenderDataPointUtil.EMPTY_DATA;
-		}
-	}
-	
-	public boolean setDataPoint(long data, int posX, int posZ, int verticalIndex)
-	{
-		this.renderDataContainer[posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex] = data;
-		return true;
-	}
-	
-	public boolean copyVerticalData(IColumnDataView newData, int posX, int posZ, boolean overwriteDataWithSameGenerationMode)
-	{
-		if (DO_SAFETY_CHECKS)
-		{
-			if (newData.size() != this.verticalDataCount)
-				throw new IllegalArgumentException("newData size not the same as this column's vertical size");
-			if (posX < 0 || posX >= SECTION_SIZE)
-				throw new IllegalArgumentException("X position is out of bounds");
-			if (posZ < 0 || posZ >= SECTION_SIZE)
-				throw new IllegalArgumentException("Z position is out of bounds");
-		}
-		
-		int dataOffset = posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount;
-		int compare = RenderDataPointUtil.compareDatapointPriority(newData.get(0), this.renderDataContainer[dataOffset]);
-		if (overwriteDataWithSameGenerationMode)
-		{
-			if (compare < 0)
-			{
-				return false;
-			}
-		}
-		else
-		{
-			if (compare <= 0)
-			{
-				return false;
-			}
-		}
-		
-		// copy the newData into this column's data
-		newData.copyTo(this.renderDataContainer, dataOffset, newData.size());
-		return true;
-	}
-	
-	
-	public long getFirstDataPoint(int posX, int posZ) { return getDataPoint(posX, posZ, 0); }
-	public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer[posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex]; }
-	
-	public long[] getVerticalDataPointArray(int posX, int posZ)
-	{
-		long[] result = new long[this.verticalDataCount];
-		int index = posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount;
-		System.arraycopy(this.renderDataContainer, index, result, 0, this.verticalDataCount);
-		return result;
-	}
+	public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer.getLong(posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex); }
 	
 	public ColumnArrayView getVerticalDataPointView(int posX, int posZ)
 	{
@@ -184,8 +150,6 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
 	public ColumnQuadView getFullQuadView() { return this.getQuadViewOverRange(0, 0, SECTION_SIZE, SECTION_SIZE); }
 	public ColumnQuadView getQuadViewOverRange(int quadX, int quadZ, int quadXSize, int quadZSize) { return new ColumnQuadView(this.renderDataContainer, SECTION_SIZE, this.verticalDataCount, quadX, quadZ, quadXSize, quadZSize); }
 	
-	public int getVerticalSize() { return this.verticalDataCount; }
-	
 	
 	
 	//=============//
@@ -357,7 +321,9 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
 	
 	@Override
 	public void close() throws Exception
-	{ /* not currently needed */ }
+	{
+		DATA_SOURCE_POOL.returnPooledDataSource(this);
+	}
 	
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java

@@ -21,12 +21,13 @@ package com.seibel.distanthorizons.core.dataObjects.render.columnViews;
 
 
 import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
 
 import java.util.Arrays;
 
 public final class ColumnArrayView implements IColumnDataView
 {
-	public final long[] data;
+	public final LongArrayList data;
 	public final int size;
 	public final int offset; // offset in longs
 	/** can be 0 if this column was created for an empty data source */
@@ -34,7 +35,7 @@ public final class ColumnArrayView implements IColumnDataView
 	
 	
 	
-	public ColumnArrayView(long[] data, int size, int offset, int vertSize)
+	public ColumnArrayView(LongArrayList data, int size, int offset, int vertSize)
 	{
 		this.data = data;
 		this.size = size;
@@ -45,9 +46,9 @@ public final class ColumnArrayView implements IColumnDataView
 	
 	
 	@Override
-	public long get(int index) { return data[index + offset]; }
+	public long get(int index) { return data.getLong(index + offset); }
 	
-	public void set(int index, long value) { data[index + offset] = value; }
+	public void set(int index, long value) { data.set(index + offset, value); }
 	
 	@Override
 	public int size() { return size; }
@@ -64,7 +65,7 @@ public final class ColumnArrayView implements IColumnDataView
 		return new ColumnArrayView(data, dataCount * vertSize, offset + dataIndexStart * vertSize, vertSize);
 	}
 	
-	public void fill(long value) { Arrays.fill(data, offset, offset + size, value); }
+	public void fill(long value) { Arrays.fill(data.elements(), offset, offset + size, value); }
 	
 	public void copyFrom(IColumnDataView source) { copyFrom(source, 0); }
 	public void copyFrom(IColumnDataView source, int outputDataIndexOffset)
@@ -82,19 +83,19 @@ public final class ColumnArrayView implements IColumnDataView
 			for (int i = 0; i < source.dataCount(); i++)
 			{
 				int outputOffset = offset + outputDataIndexOffset * vertSize + i * vertSize;
-				source.subView(i, 1).copyTo(data, outputOffset, source.verticalSize());
-				Arrays.fill(data, outputOffset + source.verticalSize(),
+				source.subView(i, 1).copyTo(data.elements(), outputOffset, source.verticalSize());
+				Arrays.fill(data.elements(), outputOffset + source.verticalSize(),
 						outputOffset + vertSize, 0);
 			}
 		}
 		else
 		{
-			source.copyTo(data, offset + outputDataIndexOffset * vertSize, source.size());
+			source.copyTo(data.elements(), offset + outputDataIndexOffset * vertSize, source.size());
 		}
 	}
 	
 	@Override
-	public void copyTo(long[] target, int offset, int size) { System.arraycopy(data, this.offset, target, offset, size); }
+	public void copyTo(long[] target, int offset, int size) { System.arraycopy(data.elements(), this.offset, target, offset, size); }
 	
 	public boolean mergeWith(ColumnArrayView source, boolean override)
 	{
@@ -170,7 +171,7 @@ public final class ColumnArrayView implements IColumnDataView
 		sb.append(" [");
 		for (int i = 0; i < size; i++)
 		{
-			sb.append(RenderDataPointUtil.toString(data[offset + i]));
+			sb.append(RenderDataPointUtil.toString(data.getLong(offset + i)));
 			if (i < size - 1)
 			{
 				sb.append(",\n");
@@ -186,15 +187,18 @@ public final class ColumnArrayView implements IColumnDataView
 		return arrayHash(data, offset, size);
 	}
 	
-	private static int arrayHash(long[] a, int offset, int length)
+	private static int arrayHash(LongArrayList a, int offset, int length)
 	{
 		if (a == null)
+		{
 			return 0;
+		}
+		
 		int result = 1;
 		int end = offset + length;
 		for (int i = offset; i < end; i++)
 		{
-			long element = a[i];
+			long element = a.getLong(i);
 			int elementHash = (int) (element ^ (element >>> 32));
 			result = 31 * result + elementHash;
 		}

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnQuadView.java

@@ -19,19 +19,24 @@
 
 package com.seibel.distanthorizons.core.dataObjects.render.columnViews;
 
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+
 public class ColumnQuadView implements IColumnDataView
 {
-	private final long[] data;
+	private final LongArrayList data;
 	private final int perColumnOffset; // per column (of columns of data) offset in longs
 	private final int xSize; // x size in datapoints
 	private final int zSize; // x size in datapoints
 	private final int offset; // offset in longs
 	private final int vertSize; // vertical size in longs
 	
-	public ColumnQuadView(long[] data, int dataZWidth, int dataVertSize, int viewXOffset, int viewZOffset, int xSize, int zSize)
+	public ColumnQuadView(LongArrayList data, int dataZWidth, int dataVertSize, int viewXOffset, int viewZOffset, int xSize, int zSize)
 	{
-		if (viewXOffset + xSize > (data.length / (dataZWidth * dataVertSize)) || viewZOffset + zSize > dataZWidth)
+		if (viewXOffset + xSize > (data.size() / (dataZWidth * dataVertSize)) || viewZOffset + zSize > dataZWidth)
+		{
 			throw new IllegalArgumentException("View is out of bounds");
+		}
+		
 		this.data = data;
 		this.xSize = xSize;
 		this.zSize = zSize;
@@ -39,7 +44,7 @@ public class ColumnQuadView implements IColumnDataView
 		this.perColumnOffset = dataZWidth * dataVertSize;
 		this.offset = viewXOffset * perColumnOffset + viewZOffset * dataVertSize;
 	}
-	private ColumnQuadView(long[] data, int perColumnOffset, int offset, int vertSize, int xSize, int zSize)
+	private ColumnQuadView(LongArrayList data, int perColumnOffset, int offset, int vertSize, int xSize, int zSize)
 	{
 		this.data = data;
 		this.perColumnOffset = perColumnOffset;
@@ -60,12 +65,12 @@ public class ColumnQuadView implements IColumnDataView
 	
 	public long get(int x, int z, int v)
 	{
-		return data[offset + x * perColumnOffset + z * vertSize + v];
+		return data.getLong(offset + x * perColumnOffset + z * vertSize + v);
 	}
 	
 	public long set(int x, int z, int v, long value)
 	{
-		return data[offset + x * perColumnOffset + z * vertSize + v] = value;
+		return data.set(offset + x * perColumnOffset + z * vertSize + v, value);
 	}
 	
 	public ColumnArrayView get(int x, int z)
@@ -82,7 +87,7 @@ public class ColumnQuadView implements IColumnDataView
 	{
 		if (singleColumn.verticalSize() != vertSize) throw new IllegalArgumentException("Vertical size of singleColumn must be equal to vertSize");
 		if (singleColumn.dataCount() != 1) throw new IllegalArgumentException("SingleColumn must contain exactly one data point");
-		singleColumn.copyTo(data, offset + x * perColumnOffset + z * vertSize, singleColumn.size());
+		singleColumn.copyTo(data.elements(), offset + x * perColumnOffset + z * vertSize, singleColumn.size());
 	}
 	
 	@Override

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java

@@ -99,7 +99,7 @@ public class FullDataToRenderDataTransformer
 		final DhSectionPos pos = fullDataSource.getSectionPos();
 		final byte dataDetail = fullDataSource.getDataDetailLevel();
 		final int vertSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(fullDataSource.getDataDetailLevel());
-		final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY());
+		final ColumnRenderSource columnSource = ColumnRenderSource.getPooledRenderSource(pos, vertSize, level.getMinY(), true);
 		if (fullDataSource.isEmpty)
 		{
 			return columnSource;
@@ -304,7 +304,7 @@ public class FullDataToRenderDataTransformer
 		int dataTotalLength = fullDataColumn.size();
 		if (dataTotalLength > columnArrayView.verticalSize())
 		{
-			ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength);
+			ColumnArrayView totalColumnData = new ColumnArrayView(new LongArrayList(new long[dataTotalLength]), dataTotalLength, 0, dataTotalLength);
 			iterateAndConvert(level, fullDataMapping, blockX, blockZ, totalColumnData, fullDataColumn);
 			columnArrayView.changeVerticalSizeFrom(totalColumnData);
 		}

core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java

@@ -0,0 +1,122 @@
+package com.seibel.distanthorizons.core.file;
+
+import com.seibel.distanthorizons.core.level.IDhLevel;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Function;
+
+/**
+ * Data sources are often very large objects and aren't used for very long.
+ * This means their frequent construction and garbage collection can result in quite a bit of GC pressure.
+ * By pooling said data sources and reusing them we can drastically reduce this GC pressure and improve
+ * performance significantly.
+ */
+public class DataSourcePool<TDataSource extends IDataSource<TDhLevel>, TDhLevel extends IDhLevel>
+{
+	private final ArrayList<TDataSource> pooledDataSources = new ArrayList<>();
+	private final ReentrantLock poolLock = new ReentrantLock();
+	
+	private final Function<DhSectionPos, TDataSource> createEmptyDatasourceFunc;
+	@Nullable
+	private final IPrepPooledDataSourceFunc<TDataSource, TDhLevel> prepDatasourceFunc;
+	
+	
+	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public DataSourcePool(Function<DhSectionPos, TDataSource> createEmptyDatasourceFunc, @Nullable IPrepPooledDataSourceFunc<TDataSource, TDhLevel> prepDatasourceFunc)
+	{
+		this.createEmptyDatasourceFunc = createEmptyDatasourceFunc;
+		this.prepDatasourceFunc = prepDatasourceFunc;
+	}
+	
+	
+	
+	//===============//
+	// pool handlers //
+	//===============//
+	
+	/** 
+	 * Returns a cleared data source.
+	 * @see DataSourcePool#getPooledSource(DhSectionPos, boolean) 
+	 */
+	public TDataSource getPooledSource(DhSectionPos pos) { return this.getPooledSource(pos, true);}
+	
+	/** @return an empty data source if non are cached */
+	public TDataSource getPooledSource(DhSectionPos pos, boolean clearData)
+	{
+		try
+		{
+			this.poolLock.lock();
+			
+			int index = this.pooledDataSources.size() - 1;
+			if (index == -1)
+			{
+				// no pooled sources exist
+				return this.createEmptyDatasourceFunc.apply(pos);
+			}
+			else
+			{
+				TDataSource dataSource = this.pooledDataSources.remove(index);
+				
+				// some data sources may want to handle prep themselves 
+				// (due to needing additional inputs than what this pool keeps track of)
+				if (this.prepDatasourceFunc != null)
+				{
+					this.prepDatasourceFunc.prepDataSource(pos, clearData, dataSource);
+				}
+				
+				return dataSource;
+			}
+		}
+		finally
+		{
+			this.poolLock.unlock();
+		}
+	}
+	
+	/**
+	 * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. 
+	 * It just means a new source must be constructed next time {@link DataSourcePool#getPooledSource} is called.
+	 */
+	public void returnPooledDataSource(TDataSource dataSource)
+	{
+		if (dataSource == null)
+		{
+			return;
+		}
+		else if (this.pooledDataSources.size() > 25)
+		{
+			return;
+		}
+		
+		try
+		{
+			this.poolLock.lock();
+			this.pooledDataSources.add(dataSource);
+		}
+		finally
+		{
+			this.poolLock.unlock();
+		}
+	}
+	
+	
+	
+	//================//
+	// helper classes //
+	//================//
+	
+	@FunctionalInterface
+	public interface IPrepPooledDataSourceFunc<TDataSource extends IDataSource<TDhLevel>, TDhLevel extends IDhLevel>
+	{
+		/** @param clearData will be false if the data will be immediately overwritten anyway */
+		void prepDataSource(DhSectionPos pos, boolean clearData, TDataSource dataSource);
+	}
+	
+}

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java

@@ -162,11 +162,11 @@ public class FullDataSourceProviderV2
 	{
 		// TODO maybe just set children update flags to true?
 		// TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway
-		return FullDataSourceV2.getPooledSource(pos, true);
+		return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true);
 	}
 	
 	@Override
-	protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.getPooledSource(pos, true); }
+	protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true); }
 	
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java

@@ -112,12 +112,13 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
 		{
 			FullDataSourceV2 fullDataSource = null;
 			ColumnRenderSource[] adjacentRenderSections = null;
+			ColumnRenderSource renderSource = null;
 			
 			try
 			{
 				// get this positions data source
 				fullDataSource = this.fullDataSourceProvider.get(this.pos);
-				ColumnRenderSource renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level);
+				renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level);
 				if (renderSource.isEmpty())
 				{
 					// nothing needs to be rendered
@@ -147,14 +148,19 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
 						fullDataSource.close();
 					}
 					
+					if (renderSource != null)
+					{
+						renderSource.close();
+					}
+					
 					if (adjacentRenderSections != null)
 					{
 						for (int i = 0; i < adjacentRenderSections.length; i++)
 						{
-							ColumnRenderSource renderSource = adjacentRenderSections[i];
-							if (renderSource != null)
+							ColumnRenderSource adjacentRenderSource = adjacentRenderSections[i];
+							if (adjacentRenderSource != null)
 							{
-								renderSource.close();
+								adjacentRenderSource.close();
 							}
 						}
 					}

core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java

@@ -120,7 +120,7 @@ public class FullDataSourceV2DTO implements IBaseDTO<DhSectionPos>
 	
 	public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException
 	{
-		FullDataSourceV2 dataSource = FullDataSourceV2.getPooledSource(this.pos, false);
+		FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(this.pos, false);
 		return this.populateDataSource(dataSource, levelWrapper);
 	}