Write custom timeout logic for DelayedDataSourceCache

This should make the code a bit more transparent vs using the CacheBuilder, plus hopefully resolve a concurrent writing issue that causes monoliths

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java

@@ -29,7 +29,8 @@ import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
 import com.seibel.distanthorizons.core.file.IDataSource;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
-import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.util.*;
@@ -54,7 +55,7 @@ import java.util.List;
  * @see FullDataSourceV1
  */
 public class FullDataSourceV2 
-		extends PhantomArrayListParent
+		extends AbstractPhantomArrayList
 		implements IDataSource<IDhLevel>, IDhApiFullDataSource
 {
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@@ -1164,6 +1165,15 @@ public class FullDataSourceV2
 	
 	
 	
+	//============//
+	// unit tests //
+	//============//
+	
+	public PhantomArrayListCheckout getPhantomArrayCheckoutForUnitTesting()
+	{ return this.pooledArraysCheckout; }
+	
+	
+	
 	//================//
 	// base overrides //
 	//================//

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java

@@ -19,20 +19,13 @@
 
 package com.seibel.distanthorizons.core.dataObjects.render;
 
-import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
-import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
-import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
-import com.seibel.distanthorizons.core.file.IDataSource;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
-import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
-import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
-import com.seibel.distanthorizons.core.util.ListUtil;
 import com.seibel.distanthorizons.coreapi.ModInfo;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnQuadView;
-import com.seibel.distanthorizons.core.level.IDhClientLevel;
 import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
 import com.seibel.distanthorizons.core.util.ColorUtil;
 import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
@@ -46,7 +39,7 @@ import java.util.concurrent.atomic.AtomicLong;
  *
  * @see RenderDataPointUtil
  */
-public class ColumnRenderSource extends PhantomArrayListParent
+public class ColumnRenderSource extends AbstractPhantomArrayList
 {
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
 	

core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java

@@ -1,9 +1,5 @@
 package com.seibel.distanthorizons.core.file.fullDatafile;
 
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalCause;
-import com.google.common.cache.RemovalNotification;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
@@ -14,6 +10,7 @@ import org.jetbrains.annotations.NotNull;
 
 import java.lang.ref.WeakReference;
 import java.util.Collections;
+import java.util.Enumeration;
 import java.util.Set;
 import java.util.concurrent.*;
 import java.util.concurrent.locks.ReentrantLock;
@@ -38,7 +35,7 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable
 	
 	
 	
-	private final Cache<Long, FullDataSourceV2> dataSourceByPosition;
+	private final ConcurrentHashMap<Long, DataSourceSavedTimePair> dataSourceByPosition = new ConcurrentHashMap<Long, DataSourceSavedTimePair>();
 	
 	/* don't let two threads load the same position at the same time */
 	protected final KeyedLockContainer<Long> saveLockContainer = new KeyedLockContainer<>();
@@ -60,16 +57,15 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable
 	public DelayedFullDataSourceSaveCache(@NotNull ISaveDataSourceFunc onSaveTimeoutAsyncFunc, int saveDelayInMs)
 	{
 		this.onSaveTimeoutAsyncFunc = onSaveTimeoutAsyncFunc;
+		
+		// we can't clean items faster than the cleanup timer fires
+		if (saveDelayInMs < CLEANUP_CHECK_TIME_IN_MS)
+		{
+			LOGGER.warn("The save delay ["+saveDelayInMs+"] shouldn't be less than the cleanup check timer interval ["+CLEANUP_CHECK_TIME_IN_MS+"].");
+		}
 		this.saveDelayInMs = saveDelayInMs;
 		
 		
-		this.dataSourceByPosition =
-			CacheBuilder.newBuilder()
-				.expireAfterAccess(this.saveDelayInMs, TimeUnit.MILLISECONDS)
-				.expireAfterWrite(this.saveDelayInMs, TimeUnit.MILLISECONDS)
-				.removalListener(this::handleDataSourceRemoval)
-				.<Long, FullDataSourceV2>build();
-		
 		SAVE_CACHE_SET.add(new WeakReference<>(this));
 	}
 	
@@ -83,61 +79,61 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable
 	 * Writing into memory is done synchronously so inputDataSource can 
 	 * be closed after this method finishes.
 	 */
-	public void writeDataSourceToMemoryAndQueueSave(FullDataSourceV2 inputDataSource)
+	public void writeDataSourceToMemoryAndQueueSave(@NotNull FullDataSourceV2 inputDataSource)
 	{
 		long inputPos = inputDataSource.getPos();
 		
-		ReentrantLock lock = this.saveLockContainer.getLockForPos(inputPos);
+		ReentrantLock lockForPos = this.saveLockContainer.getLockForPos(inputPos);
 		try
 		{
-			lock.lock();
+			lockForPos.lock();
 			
-			FullDataSourceV2 memoryDataSource = this.dataSourceByPosition.getIfPresent(inputPos);
-			if (memoryDataSource == null)
+			FullDataSourceV2 memoryDataSource;
+			
+			DataSourceSavedTimePair pair = this.dataSourceByPosition.getOrDefault(inputPos, null);
+			if (pair == null)
 			{
+				// no data currently in the memory cache for this position
 				memoryDataSource = FullDataSourceV2.createEmpty(inputPos);
-			}
-			memoryDataSource.update(inputDataSource);
-			this.dataSourceByPosition.put(inputPos, memoryDataSource);
-		}
-		finally
-		{
-			lock.unlock();
-		}
-	}
-	
-	public void handleDataSourceRemoval(RemovalNotification<Long, FullDataSourceV2> removalNotification)
-	{
-		RemovalCause cause = removalNotification.getCause();
-		if (cause == RemovalCause.EXPIRED
-			|| cause == RemovalCause.COLLECTED
-			|| cause == RemovalCause.EXPLICIT
-			|| cause == RemovalCause.SIZE)
-		{
-			// close the data source after it has expired from the cache
-			FullDataSourceV2 dataSource = removalNotification.getValue();
-			if (dataSource != null)
-			{
-				this.onSaveTimeoutAsyncFunc.saveAsync(dataSource)
-					.handle((voidObj, throwable) ->
-					{
-						try
-						{
-							dataSource.close();
-						}
-						catch (Exception e)
-						{
-							LOGGER.error("Unable to close datasource ["+ DhSectionPos.toString(dataSource.getPos()) +"], removal cause: ["+cause+"], error: ["+e.getMessage()+"].", e);
-						}
-						
-						return null;
-					});
+				pair = new DataSourceSavedTimePair(memoryDataSource);
+				this.dataSourceByPosition.put(inputPos, pair);
 			}
 			else
 			{
-				LOGGER.error("Unable to close null cached data source.");
+				memoryDataSource = pair.dataSource;
 			}
+			
+			// write the new data into memory
+			memoryDataSource.update(inputDataSource);
+			// keep track of when the last time we saved something was
+			pair.updateLastWrittenTimestamp();
 		}
+		finally
+		{
+			lockForPos.unlock();
+		}
+	}
+	
+	/** when this method is called the datasource should no longer be in the memory cache */
+	public void handleDataSourceRemoval(@NotNull FullDataSourceV2 removedDataSource)
+	{
+		this.onSaveTimeoutAsyncFunc.saveAsync(removedDataSource)
+			.handle((voidObj, throwable) ->
+			{
+				try
+				{
+					// if this close method is fired multiple times
+					// monoliths can appear due to concurrent writing to the
+					// backend arrays
+					removedDataSource.close();
+				}
+				catch (Exception e)
+				{
+					LOGGER.error("Unable to close datasource ["+ DhSectionPos.toString(removedDataSource.getPos()) +"], error: ["+e.getMessage()+"].", e);
+				}
+				
+				return null;
+			});
 	}
 	
 	
@@ -146,26 +142,36 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable
 	// List methods //
 	//==============//
 	
-	public int getUnsavedCount() { return (int)this.dataSourceByPosition.size(); }
+	public int getUnsavedCount() { return this.dataSourceByPosition.size(); }
 	
+	public void flush() { this.cleanUp(true); }
 	/** Removes everything from the memory cache and fires the {@link DelayedFullDataSourceSaveCache#onSaveTimeoutAsyncFunc} for each. */
-	public void flush()
+	public void cleanUp(boolean flushAll)
 	{
-		Set<Long> keySet = this.dataSourceByPosition.asMap().keySet();
-		for (Long pos : keySet)
+		Enumeration<Long> keyIterator = this.dataSourceByPosition.keys();
+		while (keyIterator.hasMoreElements())
 		{
-			ReentrantLock lock = this.saveLockContainer.getLockForPos(pos);
+			Long pos = keyIterator.nextElement();
+			ReentrantLock posLock = this.saveLockContainer.getLockForPos(pos);
 			try
 			{
-				lock.lock();
+				posLock.lock();
 				
-				this.dataSourceByPosition.invalidate(pos);
+				DataSourceSavedTimePair savedPair = this.dataSourceByPosition.getOrDefault(pos, null);
+				if (savedPair != null)
+				{
+					if (flushAll
+						|| savedPair.dataSourceHasTimedOut(this.saveDelayInMs))
+					{
+						this.dataSourceByPosition.remove(pos);
+						this.handleDataSourceRemoval(savedPair.dataSource);
+					}
+				}
 			}
 			finally
 			{
-				lock.unlock();
+				posLock.unlock();
 			}
-			
 		}
 	}
 	
@@ -197,7 +203,7 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable
 					}
 					else
 					{
-						cache.dataSourceByPosition.cleanUp();
+						cache.cleanUp(false);
 					}
 				});
 			}
@@ -240,4 +246,36 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable
 		CompletableFuture<Void> saveAsync(FullDataSourceV2 inputDataSource);
 	}
 	
+	/** 
+	 * used to keep track of when data sources
+	 * were written to so we can flush them once
+	 * enough time has passed.
+	 */
+	private static class DataSourceSavedTimePair
+	{
+		@NotNull
+		public final FullDataSourceV2 dataSource;
+		/** the last unix millisecond time this data source was written to */
+		public long lastWrittenDateTimeMs;
+		
+		
+		public DataSourceSavedTimePair(@NotNull FullDataSourceV2 dataSource)
+		{
+			this.dataSource = dataSource;
+			this.lastWrittenDateTimeMs = System.currentTimeMillis();
+		}
+		
+		
+		public void updateLastWrittenTimestamp()
+		{ this.lastWrittenDateTimeMs = System.currentTimeMillis(); }
+		
+		public boolean dataSourceHasTimedOut(long msTillTimeout)
+		{
+			long currentTime = System.currentTimeMillis();
+			long timeSinceUpdate = currentTime - this.lastWrittenDateTimeMs;
+			return (timeSinceUpdate > msTillTimeout);
+		}
+	}
+	
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java

@@ -16,13 +16,13 @@ import java.lang.ref.PhantomReference;
  * @see PhantomArrayListCheckout
  * @see PhantomArrayListPool
  */
-public abstract class PhantomArrayListParent implements AutoCloseable
+public abstract class AbstractPhantomArrayList implements AutoCloseable
 {
 	private static final Logger LOGGER = LogManager.getLogger();
 	
 	
 	private final PhantomArrayListPool phantomArrayListPool;
-	private final PhantomReference<PhantomArrayListParent> phantomReference;
+	private final PhantomReference<AbstractPhantomArrayList> phantomReference;
 
 	/** 
 	 * It's recommended to set this as null after the child's constructor 
@@ -37,7 +37,7 @@ public abstract class PhantomArrayListParent implements AutoCloseable
 	//=============//
 	
 	/** The Array counts can be 0 or greater. */
-	public PhantomArrayListParent(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount) 
+	public AbstractPhantomArrayList(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount) 
 	{
 		if (byteArrayCount < 0 || shortArrayCount < 0 || longArrayCount < 0)
 		{

core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java

@@ -10,13 +10,12 @@ import org.jetbrains.annotations.Nullable;
 
 import java.lang.ref.SoftReference;
 import java.util.ArrayList;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * This keeps track of all the poolable
  * arrays that can be retrieved via the {@link PhantomArrayListPool}.
  * 
- * @see PhantomArrayListParent
+ * @see AbstractPhantomArrayList
  * @see PhantomArrayListPool
  */
 public class PhantomArrayListCheckout implements AutoCloseable

core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java

@@ -33,18 +33,18 @@ import java.util.concurrent.atomic.AtomicInteger;
  * we pool these arrays when possible. <br><br>
  * 
  * How pooled arrays can be returned: <br>
- * 1. <b> Closing the {@link PhantomArrayListParent} </b> <br>
+ * 1. <b> Closing the {@link AbstractPhantomArrayList} </b> <br>
  * The fastest and most efficient method of returning pooled arrays
  * is to call {@link AutoCloseable#close()}. <br><br>
  * 
- * 2. <b> {@link PhantomArrayListParent} Garbage Collection </b> <br>
+ * 2. <b> {@link AbstractPhantomArrayList} Garbage Collection </b> <br>
  * Some objects are used across many different threads and
- * cleanly closing them is impossible, so when the {@link PhantomArrayListParent}
+ * cleanly closing them is impossible, so when the {@link AbstractPhantomArrayList}
  * is automatically garbage collected we recover and recycle any
  * arrays it checked out.
  * This is less efficient since it may allow a lot of additional arrays to
  * be created while we wait for the garbage collector to run, but 
- * does prevent any leaks from {@link PhantomArrayListParent} that weren't closed.
+ * does prevent any leaks from {@link AbstractPhantomArrayList} that weren't closed.
  * 
  * <br><br>
  * <strong>Use Notes: </strong><br>
@@ -83,9 +83,9 @@ public class PhantomArrayListPool
 	 */
 	public final boolean logGarbageCollectedStacks;
 	
-	public final ConcurrentHashMap<Reference<? extends PhantomArrayListParent>, PhantomArrayListCheckout>
+	public final ConcurrentHashMap<Reference<? extends AbstractPhantomArrayList>, PhantomArrayListCheckout>
 			phantomRefToCheckout = new ConcurrentHashMap<>();
-	public final ReferenceQueue<PhantomArrayListParent> phantomRefQueue = new ReferenceQueue<>();
+	public final ReferenceQueue<AbstractPhantomArrayList> phantomRefQueue = new ReferenceQueue<>();
 	
 	
 	private final ConcurrentLinkedQueue<SoftReference<PhantomArrayListCheckout>> pooledCheckoutsRefs = new ConcurrentLinkedQueue<>();
@@ -274,7 +274,7 @@ public class PhantomArrayListPool
 					
 					allocationStackTraceCountPairList.clear();
 					
-					Reference<? extends PhantomArrayListParent> phantomRef = pool.phantomRefQueue.poll();
+					Reference<? extends AbstractPhantomArrayList> phantomRef = pool.phantomRefQueue.poll();
 					while (phantomRef != null)
 					{
 						// return the pooled arrays
@@ -378,7 +378,7 @@ public class PhantomArrayListPool
 	// return checkout //
 	//=================//
 	
-	public void returnParentPhantomRef(@NotNull PhantomReference<PhantomArrayListParent> parentRef)
+	public void returnParentPhantomRef(@NotNull PhantomReference<AbstractPhantomArrayList> parentRef)
 	{
 		try
 		{

core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java

@@ -25,7 +25,7 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
 import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
 import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
-import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.network.INetworkObject;
@@ -47,7 +47,7 @@ import java.io.*;
 
 /** handles storing {@link FullDataSourceV2}'s in the database. */
 public class FullDataSourceV2DTO 
-		extends PhantomArrayListParent
+		extends AbstractPhantomArrayList
 		implements IBaseDTO<Long>, INetworkObject, AutoCloseable
 {
 	public static final boolean VALIDATE_INPUT_DATAPOINTS = true;

core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java

@@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.util;
 import com.google.common.annotations.VisibleForTesting;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
 import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView;
-import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
+import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
 import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
 import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
@@ -46,7 +46,7 @@ import it.unimi.dsi.fastutil.shorts.ShortArrays;
  *
  * @author Builderb0y
  */
-public class RenderDataPointReducingList extends PhantomArrayListParent
+public class RenderDataPointReducingList extends AbstractPhantomArrayList
 {
 
 	/**

core/src/test/java/tests/DelayedSaveCacheTest.java

@@ -0,0 +1,138 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package tests;
+
+import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import com.seibel.distanthorizons.core.file.fullDatafile.DelayedFullDataSourceSaveCache;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import com.seibel.distanthorizons.core.util.KeyedLockContainer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * A few very basic tests to confirm {@link DelayedFullDataSourceSaveCache}
+ * is working properly.
+ *
+ * @author James Seibel
+ * @version 2025-10-02
+ */
+public class DelayedSaveCacheTest
+{
+	
+	
+	
+	@Test
+	public void CacheExpirationAndPoolingTest() throws InterruptedException
+	{
+		// how many times any data source has been "written to disk"
+		AtomicInteger diskSaveCountRef = new AtomicInteger(0);
+		
+		DelayedFullDataSourceSaveCache cache = new DelayedFullDataSourceSaveCache((FullDataSourceV2 fullDataSource) -> 
+				{
+					diskSaveCountRef.getAndIncrement();
+					return this.onDataSourceSaveAsync(fullDataSource);
+				}, 1_000);
+		
+		
+		
+		//==============================//
+		// single item and manual flush //
+		//==============================//
+		
+		PhantomArrayListCheckout initialCheckout;
+		try (FullDataSourceV2 initialSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte)6, 0, 0)))
+		{
+			initialCheckout = initialSource.getPhantomArrayCheckoutForUnitTesting();
+			cache.writeDataSourceToMemoryAndQueueSave(initialSource);
+		}
+		Assert.assertEquals("only 1 item should be in the cache", 1, cache.getUnsavedCount());
+		Assert.assertEquals("no disk saves should have happened yet", 0, diskSaveCountRef.get());
+		
+		// manual flush
+		cache.flush();
+		Assert.assertEquals("memory cache should be empty after", 0, cache.getUnsavedCount());
+		Assert.assertEquals("1 manual flush was expected", 1, diskSaveCountRef.get());
+		
+		
+		
+		//======================//
+		// quick group position //
+		//======================//
+		
+		// write multiple items for the same position
+		for (int i = 0; i < 4; i++)
+		{
+			try (FullDataSourceV2 loopSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0)))
+			{
+				PhantomArrayListCheckout loopCheckout = loopSource.getPhantomArrayCheckoutForUnitTesting();
+				Assert.assertEquals(initialCheckout, loopCheckout);
+				
+				cache.writeDataSourceToMemoryAndQueueSave(loopSource);
+			}
+		}
+		// each item writes to the same place
+		Assert.assertEquals("exactly 1 item should be in the cache", 1, cache.getUnsavedCount());
+		Assert.assertEquals("no new saves should have happened yet", 1, diskSaveCountRef.get());
+		
+		// wait for the cache to clear
+		Thread.sleep(2_000);
+		Assert.assertEquals("Cache should have automatically cleared due to inactivity", 0, cache.getUnsavedCount());
+		Assert.assertEquals("second save after timeout expected", 2, diskSaveCountRef.get());
+		
+		
+		
+		//=====================//
+		// slow group position //
+		//=====================//
+		
+		// write multiple items for the same position
+		for (int i = 0; i < 4; i++)
+		{
+			try (FullDataSourceV2 loopSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0)))
+			{
+				PhantomArrayListCheckout loopCheckout = loopSource.getPhantomArrayCheckoutForUnitTesting();
+				Assert.assertEquals(initialCheckout, loopCheckout);
+				
+				cache.writeDataSourceToMemoryAndQueueSave(loopSource);
+			}
+			
+			// long enough to prevent a timeout, but short enough that they don't happen all at once
+			Thread.sleep(500);
+		}
+		// each item writes to the same place
+		Assert.assertEquals("exactly 1 item should be in the cache", 1, cache.getUnsavedCount());
+		Assert.assertEquals("no new saves should have happened yet", 2, diskSaveCountRef.get());
+		
+		// wait for the cache to clear
+		Thread.sleep(2_000);
+		Assert.assertEquals("Cache should have automatically cleared due to inactivity", 0, cache.getUnsavedCount());
+		Assert.assertEquals("third timeout expected", 3, diskSaveCountRef.get());
+		
+	}
+	private CompletableFuture<Void> onDataSourceSaveAsync(FullDataSourceV2 fullDataSource)
+	{ return CompletableFuture.completedFuture(null); }
+	
+}

core/src/test/java/tests/KeyedLockTest.java

@@ -0,0 +1,65 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package tests;
+
+import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import com.seibel.distanthorizons.core.util.KeyedLockContainer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * @see KeyedLockContainer
+ *
+ * @author James Seibel
+ * @version 2025-10-02
+ */
+public class KeyedLockTest
+{
+	
+	@Test
+	public void BasicKeyedLockTest()
+	{
+		KeyedLockContainer<Long> lockContainer = new KeyedLockContainer<>();
+		
+		for (long a = -10; a < 10; a++)
+		{
+			ReentrantLock aLock = lockContainer.getLockForPos(a);
+			
+			for (long b = -10; b < 10; b++)
+			{
+				ReentrantLock bLock = lockContainer.getLockForPos(a);
+				
+				// we only care that the same position always map to the same object
+				// if different positions map to the same object,
+				// that's expected hash-collision behavior and is fine
+				if (a == b)
+				{
+					Assert.assertEquals("long values ["+a+"] and ["+b+"] should have returned the same lock", aLock, bLock);
+				}
+			}
+		}
+		
+	}
+	
+}

core/src/test/java/tests/PooledDataSourceCheckoutTest.java

@@ -0,0 +1,66 @@
+/*
+ *    This file is part of the Distant Horizons mod
+ *    licensed under the GNU LGPL v3 License.
+ *
+ *    Copyright (C) 2020 James Seibel
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU Lesser General Public License as published by
+ *    the Free Software Foundation, version 3.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Lesser General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Lesser General Public License
+ *    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package tests;
+
+import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
+import com.seibel.distanthorizons.core.file.fullDatafile.DelayedFullDataSourceSaveCache;
+import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
+import com.seibel.distanthorizons.core.pos.DhSectionPos;
+import com.seibel.distanthorizons.core.util.KeyedLockContainer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * @see PhantomArrayListCheckout
+ *
+ * @author James Seibel
+ * @version 2025-10-02
+ */
+public class PooledDataSourceCheckoutTest
+{
+	
+	@Test
+	public void TestCheckouts()
+	{
+		PhantomArrayListCheckout initialCheckout;
+		try (FullDataSourceV2 initialSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte)6, 0, 0)))
+		{
+			initialCheckout = initialSource.getPhantomArrayCheckoutForUnitTesting();
+		}
+		
+		try (FullDataSourceV2 outerSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0)))
+		{
+			PhantomArrayListCheckout outerCheckout = outerSource.getPhantomArrayCheckoutForUnitTesting();
+			Assert.assertEquals("the first checkout object should be pooled", initialCheckout, outerCheckout);
+			
+			try (FullDataSourceV2 innerSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0)))
+			{
+				PhantomArrayListCheckout innerCheckout = innerSource.getPhantomArrayCheckoutForUnitTesting();
+				Assert.assertNotEquals("the second checkout object should not be shared when the first is still in use", initialCheckout, innerCheckout);
+				Assert.assertNotEquals("the second checkout object should not be shared when the first is still in use", outerCheckout, innerCheckout);
+			}
+		}
+	}
+	
+}