Massively optimize array pooling

Separating pools for each object/use case prevents infinitely growing arrays (also the column render source fix in 481e0411ac prevents infinitely allocating arrays)

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java

@@ -30,6 +30,7 @@ import com.seibel.distanthorizons.core.file.IDataSource;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.util.*;
 import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
@@ -72,6 +73,8 @@ public class FullDataSourceV2
 	
 	public static final byte DATA_FORMAT_VERSION = 1;
 	
+	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("FullDataV2");
+	
 	
 	
 	private int cachedHashCode = 0;
@@ -216,7 +219,7 @@ public class FullDataSourceV2
 			@Nullable byte[] columnGenerationSteps, @Nullable byte[] columnWorldCompressionMode,
 			boolean empty)
 	{
-		super(2, 0, WIDTH * WIDTH);
+		super(ARRAY_LIST_POOL, 2, 0, WIDTH * WIDTH);
 		
 		LodUtil.assertTrue(data == null || data.length == WIDTH * WIDTH);
 		
@@ -246,7 +249,7 @@ public class FullDataSourceV2
 		}
 		
 		// pooled generation step array
-		this.columnGenerationSteps = this.pooledArraysCheckout.getByteArray(0);
+		this.columnGenerationSteps = this.pooledArraysCheckout.getByteArray(0, 0); // initial size is 0 so we can simply add the existing array if present
 		if (columnGenerationSteps != null)
 		{
 			this.columnGenerationSteps.addElements(0, columnGenerationSteps);
@@ -257,7 +260,7 @@ public class FullDataSourceV2
 		}
 		
 		// pooled column compression array
-		this.columnWorldCompressionMode = this.pooledArraysCheckout.getByteArray(1);
+		this.columnWorldCompressionMode = this.pooledArraysCheckout.getByteArray(1, 0);
 		if (columnWorldCompressionMode != null)
 		{
 			this.columnWorldCompressionMode.addElements(0, columnWorldCompressionMode);

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java

@@ -25,8 +25,10 @@ import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRender
 import com.seibel.distanthorizons.core.file.IDataSource;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import com.seibel.distanthorizons.core.util.ListUtil;
 import com.seibel.distanthorizons.coreapi.ModInfo;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnQuadView;
@@ -55,6 +57,8 @@ public class ColumnRenderSource
 	/** width of this data in columns */
 	public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); // 64
 	
+	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Render Source");
+	
 	
 	
 	/** will be zero if an empty data source was created */
@@ -62,7 +66,7 @@ public class ColumnRenderSource
 	public long pos;
 	public int yOffset;
 	
-	public LongArrayList renderDataContainer;
+	public final LongArrayList renderDataContainer;
 	
 	public final DebugSourceFlag[] debugSourceFlags;
 	
@@ -86,7 +90,7 @@ public class ColumnRenderSource
 	 */
 	private ColumnRenderSource(long pos, int maxVerticalSize, int yOffset)
 	{
-		super(0, 0, 1);
+		super(ARRAY_LIST_POOL, 0, 0, 1);
 		
 		this.pos = pos;
 		this.yOffset = yOffset;

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java

@@ -57,6 +57,8 @@ public class FullDataToRenderDataTransformer
 	
 	private static final LongOpenHashSet brokenPos = new LongOpenHashSet();
 	
+	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Data Transformer");
+	
 	
 	
 	//==============================//
@@ -163,7 +165,7 @@ public class FullDataToRenderDataTransformer
 		else
 		{
 			// new LongArrayList(new long[fullDataLength])
-			PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(0, 0, 1);
+			PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutArrays(0, 0, 1);
 			LongArrayList dataArrayList = checkout.getLongArray(0, fullDataLength);
 			
 			try
@@ -175,7 +177,7 @@ public class FullDataToRenderDataTransformer
 			}
 			finally
 			{
-				PhantomArrayListPool.INSTANCE.returnCheckout(checkout);
+				ARRAY_LIST_POOL.returnCheckout(checkout);
 			}
 		}
 	}

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java

@@ -297,6 +297,9 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 				.noneMatch(i -> columnGenerationSteps.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value);
 	}
 	
+	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Generated Provider");
+	
+	
 	@Override
 	public LongArrayList getPositionsToRetrieve(Long pos)
 	{
@@ -310,9 +313,9 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 		// don't check any child positions if this position is already fully generated 
 		if (this.repo.existsWithKey(pos))
 		{
-			try(PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(1, 0, 0))
+			try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutArrays(1, 0, 0))
 			{
-				ByteArrayList columnGenStepArray = checkout.getByteArray(0);
+				ByteArrayList columnGenStepArray = checkout.getByteArray(0, FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH);
 				this.repo.getColumnGenerationStepForPos(pos, columnGenStepArray);
 				if (!columnGenStepArray.isEmpty())
 				{
@@ -357,9 +360,9 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
 			{
 				
 				EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT;
-				try(PhantomArrayListCheckout checkout = PhantomArrayListPool.INSTANCE.checkoutArrays(1, 0, 0))
+				try(PhantomArrayListCheckout checkout = ARRAY_LIST_POOL.checkoutArrays(1, 0, 0))
 				{
-					ByteArrayList columnGenerationSteps = checkout.getByteArray(0);
+					ByteArrayList columnGenerationSteps = checkout.getByteArray(0, FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH);
 					this.repo.getColumnGenerationStepForPos(genPos, columnGenerationSteps);
 					if (columnGenerationSteps.isEmpty())
 					{

core/src/main/java/com/seibel/distanthorizons/core/logging/DhLoggerBuilder.java

@@ -59,6 +59,12 @@ public class DhLoggerBuilder
 	
 	/** Attempts to return the logger for this containing class. */
 	public static Logger getLogger()
+	{
+		return LogManager.getLogger(ModInfo.NAME + "-" + getCallingClassName());
+	}
+	
+	/** @return "??" if no name could be found */
+	public static String getCallingClassName()
 	{
 		StackTraceElement[] stElements = Thread.currentThread().getStackTrace();
 		String callerClassName = "??";
@@ -72,7 +78,9 @@ public class DhLoggerBuilder
 				break;
 			}
 		}
-		return LogManager.getLogger(ModInfo.NAME + "-" + callerClassName);
+		
+		return callerClassName;
 	}
 	
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java

@@ -108,7 +108,7 @@ public class F3Screen
 		messageList.add(getThreadPoolStatString("Buffer Builder", bufferBuilderPool));
 		messageList.add("");
 		// object pools
-		PhantomArrayListPool.INSTANCE.addDebugMenuStringsToList(messageList);
+		PhantomArrayListPool.addDebugMenuStringsToListForCombinedPools(messageList);
 		messageList.add("");
 		// chunk updates
 		messageList.add(SharedApi.INSTANCE.getDebugMenuString());

core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java

@@ -2,6 +2,7 @@ package com.seibel.distanthorizons.core.pooling;
 
 import com.seibel.distanthorizons.core.util.ListUtil;
 import it.unimi.dsi.fastutil.bytes.ByteArrayList;
+import it.unimi.dsi.fastutil.ints.Int2ReferenceArrayMap;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.shorts.ShortArrayList;
 
@@ -16,12 +17,26 @@ import java.util.ArrayList;
  */
 public class PhantomArrayListCheckout implements AutoCloseable
 {
+	/** defines which pool the arrays should be returned too */
+	private final PhantomArrayListPool owningPool;
+	
 	private final ArrayList<ByteArrayList> byteArrayLists = new ArrayList<>();
 	private final ArrayList<ShortArrayList> shortArrayLists = new ArrayList<>();
 	private final ArrayList<LongArrayList> longArrayLists = new ArrayList<>();
 	
 	
 	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public PhantomArrayListCheckout(PhantomArrayListPool owningPool)
+	{
+		this.owningPool = owningPool;
+	}
+	
+	
+	
 	//=========//
 	// setters //
 	//=========//
@@ -42,62 +57,41 @@ public class PhantomArrayListCheckout implements AutoCloseable
 	
 	
 	
-	public ByteArrayList getByteArray(int index) { return this.getByteArray(index, 0); }
 	public ByteArrayList getByteArray(int index, int size)
 	{
 		ByteArrayList list = this.byteArrayLists.get(index);
-		if (size != 0)
-		{
-			ListUtil.clearAndSetSize(list, size);
-		}
-		else
-		{
-			list.clear();
-		}
+		ListUtil.clearAndSetSize(list, size);
 		return list;
 	}
-	
-	public ShortArrayList getShortArray(int index) { return this.getShortArray(index, 0); }
 	public ShortArrayList getShortArray(int index, int size)
 	{
 		ShortArrayList list = this.shortArrayLists.get(index);
-		if (size != 0)
-		{
-			ListUtil.clearAndSetSize(list, size);
-		}
-		else
-		{
-			list.clear();
-		}
+		ListUtil.clearAndSetSize(list, size);
 		return list;
 	}
-	
-	public LongArrayList getLongArray(int index) { return this.getLongArray(index, 0); }
 	public LongArrayList getLongArray(int index, int size)
 	{
 		LongArrayList list = this.longArrayLists.get(index);
-		if (size != 0)
-		{
-			ListUtil.clearAndSetSize(list, size);
-		}
-		else
-		{
-			list.clear();
-		}
+		ListUtil.clearAndSetSize(list, size);
 		return list;
 	}
 	
-	
 	public ArrayList<ByteArrayList> getAllByteArrays() { return this.byteArrayLists; }
 	public ArrayList<ShortArrayList> getAllShortArrays() { return this.shortArrayLists; }
 	public ArrayList<LongArrayList> getAllLongArrays() { return this.longArrayLists; }
 	
 	
 	
+	//================//
+	// base overrides //
+	//================//
+	
 	@Override 
 	public void close()
 	{
-		PhantomArrayListPool.INSTANCE.returnCheckout(this);
+		this.owningPool.returnCheckout(this);
 	}
 	
+	
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java

@@ -1,14 +1,9 @@
 package com.seibel.distanthorizons.core.pooling;
 
-import com.seibel.distanthorizons.core.util.ThreadUtil;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.lang.ref.PhantomReference;
-import java.lang.ref.Reference;
-import java.lang.ref.ReferenceQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadPoolExecutor;
 
 /**
  * Any object that needs pooled arrays should extend this object.
@@ -25,15 +20,8 @@ public abstract class PhantomArrayListParent implements AutoCloseable
 {
 	private static final Logger LOGGER = LogManager.getLogger();
 	
-	private static final ConcurrentHashMap<Reference<? extends PhantomArrayListParent>, PhantomArrayListCheckout>
-			PHANTOM_REF_TO_CHECKOUT = new ConcurrentHashMap<>();
-	private static final ReferenceQueue<PhantomArrayListParent> PHANTOM_REF_QUEUE = new ReferenceQueue<>();
-	
-	private static final int PHANTOM_REF_CHECK_TIME_IN_MS = 5 * 1000;
-	private static final ThreadPoolExecutor RECYCLER_THREAD = ThreadUtil.makeSingleThreadPool("Phantom Array Recycler");
-	
-	
 	
+	private final PhantomArrayListPool phantomArrayListPool;
 	private final PhantomReference<PhantomArrayListParent> phantomReference;
 
 	/** 
@@ -44,82 +32,22 @@ public abstract class PhantomArrayListParent implements AutoCloseable
 	
 	
 	
-	//====================//
-	// static constructor //
-	//====================//
-	
-	static { RECYCLER_THREAD.execute(() -> runPhantomReferenceCleanupLoop()); }
-	
-	private static void runPhantomReferenceCleanupLoop()
-	{
-		while (true)
-		{
-			try
-			{
-				try
-				{
-					Thread.sleep(PHANTOM_REF_CHECK_TIME_IN_MS);
-				}
-				catch (InterruptedException ignore) { }
-				
-				
-				int returnedByteArrayCount = 0;
-				int returnedShortArrayCount = 0;
-				int returnedLongArrayCount = 0;
-				Reference<? extends PhantomArrayListParent> phantomRef = PHANTOM_REF_QUEUE.poll();
-				while (phantomRef != null)
-				{
-					// return the pooled arrays
-					PhantomArrayListCheckout checkout = PHANTOM_REF_TO_CHECKOUT.remove(phantomRef);
-					if (checkout != null)
-					{
-						returnedByteArrayCount += checkout.getByteArrayCount();
-						returnedShortArrayCount += checkout.getShortArrayCount();
-						returnedLongArrayCount += checkout.getLongArrayCount();
-						PhantomArrayListPool.INSTANCE.returnCheckout(checkout);
-					}
-					else
-					{
-						// shouldn't happen, but just in case
-						LOGGER.warn("Unable to find checkout for phantom reference ["+phantomRef+"], arrays will need to be recreated.");
-					}
-					
-					phantomRef = PHANTOM_REF_QUEUE.poll();
-				}
-				
-				if (returnedByteArrayCount != 0 && returnedLongArrayCount != 0)
-				{
-					// we only want to log when arrays have been returned
-					//LOGGER.info("Returned byte:["+F3Screen.NUMBER_FORMAT.format(returnedByteArrayCount)+"], short:["+F3Screen.NUMBER_FORMAT.format(returnedShortArrayCount)+"], long:["+F3Screen.NUMBER_FORMAT.format(returnedLongArrayCount)+"].");
-					
-					// since this is just for debugging it only needs to be recalculated once in a while
-					PhantomArrayListPool.INSTANCE.recalculateSizeForDebugging();
-				}
-			}
-			catch (Exception e)
-			{
-				LOGGER.error("Unexpected error in phantom pool return thread, error: [" + e.getMessage() + "].", e);
-			}
-		}
-	}
-	
-	
-	
 	//=============//
 	// constructor //
 	//=============//
 	
 	/** The Array counts can be 0 or greater. */
-	public PhantomArrayListParent(int byteArrayCount, int shortArrayCount, int longArrayCount) 
+	public PhantomArrayListParent(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount) 
 	{
 		if (byteArrayCount < 0 || shortArrayCount < 0 || longArrayCount < 0)
 		{
 			throw new IllegalArgumentException("Can't get a negative number of pooled arrays.");
 		}
 		
-		this.phantomReference = new PhantomReference<>(this, PHANTOM_REF_QUEUE);
-		this.pooledArraysCheckout = PhantomArrayListPool.INSTANCE.checkoutArrays(byteArrayCount, shortArrayCount, longArrayCount);
-		PHANTOM_REF_TO_CHECKOUT.put(this.phantomReference, this.pooledArraysCheckout);
+		this.phantomArrayListPool = phantomArrayListPool;
+		this.phantomReference = new PhantomReference<>(this, this.phantomArrayListPool.phantomRefQueue);
+		this.pooledArraysCheckout = this.phantomArrayListPool.checkoutArrays(byteArrayCount, shortArrayCount, longArrayCount);
+		this.phantomArrayListPool.phantomRefToCheckout.put(this.phantomReference, this.pooledArraysCheckout);
 	}
 	
 	
@@ -134,8 +62,8 @@ public abstract class PhantomArrayListParent implements AutoCloseable
 		try
 		{
 			this.phantomReference.clear();
-			PhantomArrayListCheckout checkout = PHANTOM_REF_TO_CHECKOUT.remove(this.phantomReference);
-			PhantomArrayListPool.INSTANCE.returnCheckout(checkout);
+			PhantomArrayListCheckout checkout = this.phantomArrayListPool.phantomRefToCheckout.remove(this.phantomReference);
+			this.phantomArrayListPool.returnCheckout(checkout);
 		}
 		catch (Exception e)
 		{

core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java

@@ -2,15 +2,20 @@ package com.seibel.distanthorizons.core.pooling;
 
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.logging.f3.F3Screen;
+import com.seibel.distanthorizons.core.util.ThreadUtil;
 import com.seibel.distanthorizons.coreapi.util.StringUtil;
 import it.unimi.dsi.fastutil.bytes.ByteArrayList;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.shorts.ShortArrayList;
 import org.apache.logging.log4j.Logger;
 
+import java.lang.ref.Reference;
+import java.lang.ref.ReferenceQueue;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Supplier;
@@ -38,7 +43,27 @@ public class PhantomArrayListPool
 {
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	
-	public static final PhantomArrayListPool INSTANCE = new PhantomArrayListPool();
+	/** 
+	 * the recycler thread needs to be triggered relatively often to prevent
+	 * build up of GC'ed arrays.
+	 */
+	private static final int PHANTOM_REF_CHECK_TIME_IN_MS = 1_000;
+	private static final ThreadPoolExecutor RECYCLER_THREAD = ThreadUtil.makeSingleThreadPool("Phantom Array Recycler");
+	private static final ArrayList<PhantomArrayListPool> POOL_LIST = new ArrayList<>();
+	
+	/** if enabled the number of GC'ed arrays will be logged */
+	private static final boolean LOG_ARRAY_RECOVERY = false;
+	
+	
+	
+	/** used for debugging and tracking what the pool contains */
+	public final String name;
+	
+	public final ConcurrentHashMap<Reference<? extends PhantomArrayListParent>, PhantomArrayListCheckout>
+			phantomRefToCheckout = new ConcurrentHashMap<>();
+	public final ReferenceQueue<PhantomArrayListParent> phantomRefQueue = new ReferenceQueue<>();
+	
+	
 	
 	
 	/** needed since our pools aren't thread safe */
@@ -68,12 +93,82 @@ public class PhantomArrayListPool
 	// constructor //
 	//=============//
 	
-	private PhantomArrayListPool()
+	// shared setup used by all pools
+	static
 	{
-		// create initial arrays
-		PhantomArrayListCheckout checkout = this.checkoutArrays(2_000, 100, 100_000);
-		this.returnCheckout(checkout);
+		RECYCLER_THREAD.execute(() -> runPhantomReferenceCleanupLoop());
 	}
+	private static void runPhantomReferenceCleanupLoop()
+	{
+		while (true)
+		{
+			try
+			{
+				try
+				{
+					Thread.sleep(PHANTOM_REF_CHECK_TIME_IN_MS);
+				}
+				catch (InterruptedException ignore) { }
+				
+				
+				for (int i = 0; i < POOL_LIST.size(); i++)
+				{
+					PhantomArrayListPool pool = POOL_LIST.get(i);
+					
+					int returnedByteArrayCount = 0;
+					int returnedShortArrayCount = 0;
+					int returnedLongArrayCount = 0;
+					Reference<? extends PhantomArrayListParent> phantomRef = pool.phantomRefQueue.poll();
+					while (phantomRef != null)
+					{
+						// return the pooled arrays
+						PhantomArrayListCheckout checkout = pool.phantomRefToCheckout.remove(phantomRef);
+						if (checkout != null)
+						{
+							returnedByteArrayCount += checkout.getByteArrayCount();
+							returnedShortArrayCount += checkout.getShortArrayCount();
+							returnedLongArrayCount += checkout.getLongArrayCount();
+							pool.returnCheckout(checkout);
+						}
+						else
+						{
+							// shouldn't happen, but just in case
+							LOGGER.warn("Pool: ["+pool.name+"]. Unable to find checkout for phantom reference ["+phantomRef+"], arrays will need to be recreated.");
+						}
+						
+						phantomRef = pool.phantomRefQueue.poll();
+					}
+					
+					if (LOG_ARRAY_RECOVERY)
+					{
+						if (returnedByteArrayCount != 0
+								&& returnedShortArrayCount != 0
+								&& returnedLongArrayCount != 0)
+						{
+							// we only want to log when arrays have been returned
+							LOGGER.info("Pool: ["+pool.name+"]. Returned byte:["+F3Screen.NUMBER_FORMAT.format(returnedByteArrayCount)+"], short:["+F3Screen.NUMBER_FORMAT.format(returnedShortArrayCount)+"], long:["+F3Screen.NUMBER_FORMAT.format(returnedLongArrayCount)+"].");
+						}
+					}
+					
+					// since this is just for debugging it only needs to be recalculated once in a while
+					pool.recalculateSizeForDebugging();
+				}
+			}
+			catch (Exception e)
+			{
+				LOGGER.error("Unexpected error in phantom pool return thread, error: [" + e.getMessage() + "].", e);
+			}
+		}
+	}
+	
+	
+	public PhantomArrayListPool(String name)
+	{
+		POOL_LIST.add(this);
+		this.name = name;
+	}
+	
+	
 	
 	
 	
@@ -87,24 +182,24 @@ public class PhantomArrayListPool
 		{
 			this.poolLock.lock();
 			
-			PhantomArrayListCheckout checkout = new PhantomArrayListCheckout();
+			PhantomArrayListCheckout checkout = new PhantomArrayListCheckout(this);
 			
 			// byte
 			for (int i = 0; i < byteArrayCount; i++)
 			{
-				checkout.addByteArrayList(getPooledArray(this.pooledByteArrays, this::createEmptyByteArrayList));
+				checkout.addByteArrayList(getPooledArray(this.pooledByteArrays, () -> this.createEmptyByteArrayList()));
 			}
 			
 			// short
 			for (int i = 0; i < shortArrayCount; i++)
 			{
-				checkout.addShortArrayList(getPooledArray(this.pooledShortArrays, this::createEmptyShortArrayList));
+				checkout.addShortArrayList(getPooledArray(this.pooledShortArrays, () -> this.createEmptyShortArrayList()));
 			}
 			
 			// long
 			for (int i = 0; i < longArrayCount; i++)
 			{
-				checkout.addLongArrayList(getPooledArray(this.pooledLongArrays, this::createEmptyLongArrayList));
+				checkout.addLongArrayList(getPooledArray(this.pooledLongArrays, () -> this.createEmptyLongArrayList()));
 			}
 			
 			return checkout;
@@ -122,19 +217,19 @@ public class PhantomArrayListPool
 	{
 		//LOGGER.error("created new byte array");
 		this.totalByteArrayCountRef.getAndIncrement();
-		return new ByteArrayList();
+		return new ByteArrayList(0);
 	}
 	private ShortArrayList createEmptyShortArrayList()
 	{
 		//LOGGER.error("created new short array");
 		this.totalShortArrayCountRef.getAndIncrement();
-		return new ShortArrayList();
+		return new ShortArrayList(0);
 	}
-	private LongArrayList createEmptyLongArrayList()
+	public LongArrayList createEmptyLongArrayList()
 	{
 		//LOGGER.error("created new long array");
 		this.totalLongArrayCountRef.getAndIncrement();
-		return new LongArrayList();
+		return new LongArrayList(0);
 	}
 	
 	
@@ -174,7 +269,7 @@ public class PhantomArrayListPool
 		try
 		{
 			// In James' testing pooling the checkout object wasn't necessary
-			// since it is relatively small and short lived it appears
+			// since it is relatively small and short lived, thus
 			// the GC can handle quickly discarding it.
 			
 			this.pooledByteArrays.addAll(checkout.getAllByteArrays());
@@ -195,35 +290,90 @@ public class PhantomArrayListPool
 	// debug methods //
 	//===============//
 	
+	public static void addDebugMenuStringsToListForCombinedPools(List<String> messageList)
+	{
+		int totalByteArrayCount = 0, totalShortArrayCount = 0, totalLongArrayCount = 0;
+		int pooledByteArraySize = 0, pooledShortArraySize = 0, pooledLongArraySize = 0;
+		long lastBytePoolSizeInBytes = 0, lastShortPoolSizeInBytes = 0, lastLongPoolSizeInBytes = 0;
+		
+		for (int i = 0; i < POOL_LIST.size(); i++)
+		{
+			PhantomArrayListPool pool = POOL_LIST.get(i);
+			
+			totalByteArrayCount += pool.totalByteArrayCountRef.get();
+			totalShortArrayCount += pool.totalShortArrayCountRef.get();
+			totalLongArrayCount += pool.totalLongArrayCountRef.get();
+			
+			pooledByteArraySize += pool.pooledByteArrays.size();
+			pooledShortArraySize += pool.pooledShortArrays.size();
+			pooledLongArraySize += pool.pooledLongArrays.size();
+			
+			lastBytePoolSizeInBytes += pool.lastBytePoolSizeInBytes;
+			lastShortPoolSizeInBytes += pool.lastShortPoolSizeInBytes;
+			lastLongPoolSizeInBytes += pool.lastLongPoolSizeInBytes;
+		}
+		
+		addDebugMenuStringsToList(messageList,
+			"Combined",
+			totalByteArrayCount, totalShortArrayCount, totalLongArrayCount,
+			pooledByteArraySize, pooledShortArraySize, pooledLongArraySize,
+			lastBytePoolSizeInBytes, lastShortPoolSizeInBytes, lastLongPoolSizeInBytes
+		);
+	}
+	
 	public void addDebugMenuStringsToList(List<String> messageList)
+	{
+		addDebugMenuStringsToList(messageList,
+			this.name,
+			this.totalByteArrayCountRef.get(), this.totalShortArrayCountRef.get(), this.totalLongArrayCountRef.get(),
+			this.pooledByteArrays.size(), this.pooledShortArrays.size(), this.pooledLongArrays.size(),
+			this.lastBytePoolSizeInBytes, this.lastShortPoolSizeInBytes, this.lastLongPoolSizeInBytes
+		);
+	}
+	private static void addDebugMenuStringsToList(List<String> messageList, 
+			String name,
+			int totalByteArrayCount, int totalShortArrayCount, int totalLongArrayCount,
+			int numbOfByteArraysInPool, int numbOfShortArraysInPool, int numbOfLongArraysInPool,
+			long lastBytePoolSizeInBytes, long lastShortPoolSizeInBytes, long lastLongPoolSizeInBytes)
 	{
 		// total (all time created) count
-		String byteArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalByteArrayCountRef.get());
-		String shortArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalShortArrayCountRef.get());
-		String longArrayTotalCount = F3Screen.NUMBER_FORMAT.format(this.totalLongArrayCountRef.get());
+		String byteArrayTotalCount = F3Screen.NUMBER_FORMAT.format(totalByteArrayCount);
+		String shortArrayTotalCount = F3Screen.NUMBER_FORMAT.format(totalShortArrayCount);
+		String longArrayTotalCount = F3Screen.NUMBER_FORMAT.format(totalLongArrayCount);
 		
 		// inactive items in pool
-		String bytePoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledByteArrays.size());
-		String shortPoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledShortArrays.size());
-		String longPoolCount = F3Screen.NUMBER_FORMAT.format(this.pooledLongArrays.size());
+		String bytePoolCount = F3Screen.NUMBER_FORMAT.format(numbOfByteArraysInPool);
+		String shortPoolCount = F3Screen.NUMBER_FORMAT.format(numbOfShortArraysInPool);
+		String longPoolCount = F3Screen.NUMBER_FORMAT.format(numbOfLongArraysInPool);
 		
 		// pool byte size
-		String bytePoolSizeInBytes = (this.lastBytePoolSizeInBytes != -1) 
-				? " ~" + StringUtil.convertBytesToHumanReadable(this.lastBytePoolSizeInBytes)
+		String bytePoolSizeInBytes = (lastBytePoolSizeInBytes != -1)
+				? " ~" + StringUtil.convertBytesToHumanReadable(lastBytePoolSizeInBytes)
 				: "";
-		String shortPoolSizeInBytes = (this.lastShortPoolSizeInBytes != -1)
-				? " ~" + StringUtil.convertBytesToHumanReadable(this.lastShortPoolSizeInBytes)
+		String shortPoolSizeInBytes = (lastShortPoolSizeInBytes != -1)
+				? " ~" + StringUtil.convertBytesToHumanReadable(lastShortPoolSizeInBytes)
 				: "";
-		String longPoolSizeInBytes = (this.lastLongPoolSizeInBytes != -1)
-				? " ~" + StringUtil.convertBytesToHumanReadable(this.lastLongPoolSizeInBytes)
+		String longPoolSizeInBytes = (lastLongPoolSizeInBytes != -1)
+				? " ~" + StringUtil.convertBytesToHumanReadable(lastLongPoolSizeInBytes)
 				: "";
 		
-		messageList.add("Pools:");
-		messageList.add("byte[]: "+bytePoolCount+"/"+byteArrayTotalCount + bytePoolSizeInBytes);
-		messageList.add("short[]: "+shortPoolCount+"/"+shortArrayTotalCount + shortPoolSizeInBytes);
-		messageList.add("long[]: "+longPoolCount+"/"+longArrayTotalCount + longPoolSizeInBytes);
+		
+		messageList.add(name + " - Pools:");
+		if (totalByteArrayCount != 0)
+		{
+			messageList.add("byte[]: " + bytePoolCount + "/" + byteArrayTotalCount + bytePoolSizeInBytes);
+		}
+		if (totalShortArrayCount != 0)
+		{
+			messageList.add("short[]: " + shortPoolCount + "/" + shortArrayTotalCount + shortPoolSizeInBytes);
+		}
+		if (totalLongArrayCount != 0)
+		{
+			messageList.add("long[]: " + longPoolCount + "/" + longArrayTotalCount + longPoolSizeInBytes);
+		}
 	}
 	
+	
 	/**
 	 *  shouldn't be called on the render thread as it can
 	 *  take 10's of milliseconds to complete.
@@ -244,6 +394,7 @@ public class PhantomArrayListPool
 			// long
 			long longPoolByteSize = estimateMemoryUsage(this.pooledLongArrays, Long.BYTES);
 			this.lastLongPoolSizeInBytes = Math.max(longPoolByteSize, this.lastLongPoolSizeInBytes);
+			
 		}
 		finally
 		{

core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java

@@ -26,6 +26,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio
 import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.network.INetworkObject;
 import com.seibel.distanthorizons.core.util.FullDataPointUtil;
@@ -75,6 +76,9 @@ public class FullDataSourceV2DTO
 	public long createdUnixDateTime;
 	
 	
+	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("V2DTO");
+	
+	
 	
 	//==============//
 	// constructors //
@@ -109,12 +113,14 @@ public class FullDataSourceV2DTO
 	public static FullDataSourceV2DTO CreateEmptyDataSourceForDecoding() { return new FullDataSourceV2DTO(); }
 	private FullDataSourceV2DTO() 
 	{
-		super(4, 0, 0);
+		super(ARRAY_LIST_POOL, 4, 0, 0);
 		
-		this.compressedDataByteArray = this.pooledArraysCheckout.getByteArray(0);
-		this.compressedColumnGenStepByteArray = this.pooledArraysCheckout.getByteArray(1);
-		this.compressedWorldCompressionModeByteArray = this.pooledArraysCheckout.getByteArray(2);
-		this.compressedMappingByteArray = this.pooledArraysCheckout.getByteArray(3);
+		// Expected sizes here are 0 since we don't know how big these arrays need to be,
+		// they depend on compression settings and world complexity.
+		this.compressedDataByteArray = this.pooledArraysCheckout.getByteArray(0, 0);
+		this.compressedColumnGenStepByteArray = this.pooledArraysCheckout.getByteArray(1, 0);
+		this.compressedWorldCompressionModeByteArray = this.pooledArraysCheckout.getByteArray(2, 0);
+		this.compressedMappingByteArray = this.pooledArraysCheckout.getByteArray(3, 0);
 		
 		this.pooledArraysCheckout = null;
 	}

core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java

@@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.shorts.ShortArrayList;
@@ -86,6 +87,10 @@ public class RenderDataPointReducingList extends PhantomArrayListParent
 	/** the default element of {@link #links} to indicate that a node is not linked to any other nodes. */
 	public static final long DEFAULT_LINKS = -1L;
 	
+	
+	public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Render Reducer");
+	
+	
 	/**
 	 * indexes of the nodes at the ends of this list.
 	 * access these fields through the getters,
@@ -118,7 +123,7 @@ public class RenderDataPointReducingList extends PhantomArrayListParent
 	
 	public RenderDataPointReducingList(IColumnDataView view) 
 	{
-		super(0, 1, 2);
+		super(ARRAY_LIST_POOL, 0, 1, 2);
 		
 		int size = view.size();
 		if (size == 0) 
@@ -127,9 +132,9 @@ public class RenderDataPointReducingList extends PhantomArrayListParent
 			this.setHighest(NULL);
 			this.setSmallest(NULL);
 			this.setBiggest(NULL);
-			this.links = this.pooledArraysCheckout.getLongArray(0);
-			this.data = this.pooledArraysCheckout.getLongArray(1);
-			this.sortingArray = this.pooledArraysCheckout.getShortArray(0);
+			this.links = this.pooledArraysCheckout.getLongArray(0, 0);
+			this.data = this.pooledArraysCheckout.getLongArray(1, 0);
+			this.sortingArray = this.pooledArraysCheckout.getShortArray(0, 0);
 			if (ASSERTS) this.checkLinks();
 			
 			this.pooledArraysCheckout = null;