From 721124b88617099c1b37a3da7b7affd5c734d0c6 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 2 Oct 2025 20:29:26 -0500 Subject: [PATCH] Write custom timeout logic for DelayedDataSourceCache This should make the code a bit more transparent vs using the CacheBuilder, plus hopefully resolve a concurrent writing issue that causes monoliths --- .../fullData/sources/FullDataSourceV2.java | 14 +- .../render/ColumnRenderSource.java | 11 +- .../DelayedFullDataSourceSaveCache.java | 166 +++++++++++------- ...ent.java => AbstractPhantomArrayList.java} | 6 +- .../pooling/PhantomArrayListCheckout.java | 3 +- .../core/pooling/PhantomArrayListPool.java | 16 +- .../core/sql/dto/FullDataSourceV2DTO.java | 4 +- .../util/RenderDataPointReducingList.java | 4 +- .../test/java/tests/DelayedSaveCacheTest.java | 138 +++++++++++++++ core/src/test/java/tests/KeyedLockTest.java | 65 +++++++ .../tests/PooledDataSourceCheckoutTest.java | 66 +++++++ 11 files changed, 401 insertions(+), 92 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/pooling/{PhantomArrayListParent.java => AbstractPhantomArrayList.java} (86%) create mode 100644 core/src/test/java/tests/DelayedSaveCacheTest.java create mode 100644 core/src/test/java/tests/KeyedLockTest.java create mode 100644 core/src/test/java/tests/PooledDataSourceCheckoutTest.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 00516cdb9..a224bb104 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -29,7 +29,8 @@ import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; +import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.*; @@ -54,7 +55,7 @@ import java.util.List; * @see FullDataSourceV1 */ public class FullDataSourceV2 - extends PhantomArrayListParent + extends AbstractPhantomArrayList implements IDataSource, IDhApiFullDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -1164,6 +1165,15 @@ public class FullDataSourceV2 + //============// + // unit tests // + //============// + + public PhantomArrayListCheckout getPhantomArrayCheckoutForUnitTesting() + { return this.pooledArraysCheckout; } + + + //================// // base overrides // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index c107575da..27654f016 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -19,20 +19,13 @@ package com.seibel.distanthorizons.core.dataObjects.render; -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; +import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; -import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.ListUtil; import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnQuadView; -import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; @@ -46,7 +39,7 @@ import java.util.concurrent.atomic.AtomicLong; * * @see RenderDataPointUtil */ -public class ColumnRenderSource extends PhantomArrayListParent +public class ColumnRenderSource extends AbstractPhantomArrayList { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java index cca8f2486..e982bf1c2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java @@ -1,9 +1,5 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.RemovalCause; -import com.google.common.cache.RemovalNotification; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -14,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import java.lang.ref.WeakReference; import java.util.Collections; +import java.util.Enumeration; import java.util.Set; import java.util.concurrent.*; import java.util.concurrent.locks.ReentrantLock; @@ -38,7 +35,7 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable - private final Cache dataSourceByPosition; + private final ConcurrentHashMap dataSourceByPosition = new ConcurrentHashMap(); /* don't let two threads load the same position at the same time */ protected final KeyedLockContainer saveLockContainer = new KeyedLockContainer<>(); @@ -60,16 +57,15 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable public DelayedFullDataSourceSaveCache(@NotNull ISaveDataSourceFunc onSaveTimeoutAsyncFunc, int saveDelayInMs) { this.onSaveTimeoutAsyncFunc = onSaveTimeoutAsyncFunc; + + // we can't clean items faster than the cleanup timer fires + if (saveDelayInMs < CLEANUP_CHECK_TIME_IN_MS) + { + LOGGER.warn("The save delay ["+saveDelayInMs+"] shouldn't be less than the cleanup check timer interval ["+CLEANUP_CHECK_TIME_IN_MS+"]."); + } this.saveDelayInMs = saveDelayInMs; - this.dataSourceByPosition = - CacheBuilder.newBuilder() - .expireAfterAccess(this.saveDelayInMs, TimeUnit.MILLISECONDS) - .expireAfterWrite(this.saveDelayInMs, TimeUnit.MILLISECONDS) - .removalListener(this::handleDataSourceRemoval) - .build(); - SAVE_CACHE_SET.add(new WeakReference<>(this)); } @@ -83,61 +79,61 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable * Writing into memory is done synchronously so inputDataSource can * be closed after this method finishes. */ - public void writeDataSourceToMemoryAndQueueSave(FullDataSourceV2 inputDataSource) + public void writeDataSourceToMemoryAndQueueSave(@NotNull FullDataSourceV2 inputDataSource) { long inputPos = inputDataSource.getPos(); - ReentrantLock lock = this.saveLockContainer.getLockForPos(inputPos); + ReentrantLock lockForPos = this.saveLockContainer.getLockForPos(inputPos); try { - lock.lock(); + lockForPos.lock(); - FullDataSourceV2 memoryDataSource = this.dataSourceByPosition.getIfPresent(inputPos); - if (memoryDataSource == null) + FullDataSourceV2 memoryDataSource; + + DataSourceSavedTimePair pair = this.dataSourceByPosition.getOrDefault(inputPos, null); + if (pair == null) { + // no data currently in the memory cache for this position memoryDataSource = FullDataSourceV2.createEmpty(inputPos); - } - memoryDataSource.update(inputDataSource); - this.dataSourceByPosition.put(inputPos, memoryDataSource); - } - finally - { - lock.unlock(); - } - } - - public void handleDataSourceRemoval(RemovalNotification removalNotification) - { - RemovalCause cause = removalNotification.getCause(); - if (cause == RemovalCause.EXPIRED - || cause == RemovalCause.COLLECTED - || cause == RemovalCause.EXPLICIT - || cause == RemovalCause.SIZE) - { - // close the data source after it has expired from the cache - FullDataSourceV2 dataSource = removalNotification.getValue(); - if (dataSource != null) - { - this.onSaveTimeoutAsyncFunc.saveAsync(dataSource) - .handle((voidObj, throwable) -> - { - try - { - dataSource.close(); - } - catch (Exception e) - { - LOGGER.error("Unable to close datasource ["+ DhSectionPos.toString(dataSource.getPos()) +"], removal cause: ["+cause+"], error: ["+e.getMessage()+"].", e); - } - - return null; - }); + pair = new DataSourceSavedTimePair(memoryDataSource); + this.dataSourceByPosition.put(inputPos, pair); } else { - LOGGER.error("Unable to close null cached data source."); + memoryDataSource = pair.dataSource; } + + // write the new data into memory + memoryDataSource.update(inputDataSource); + // keep track of when the last time we saved something was + pair.updateLastWrittenTimestamp(); } + finally + { + lockForPos.unlock(); + } + } + + /** when this method is called the datasource should no longer be in the memory cache */ + public void handleDataSourceRemoval(@NotNull FullDataSourceV2 removedDataSource) + { + this.onSaveTimeoutAsyncFunc.saveAsync(removedDataSource) + .handle((voidObj, throwable) -> + { + try + { + // if this close method is fired multiple times + // monoliths can appear due to concurrent writing to the + // backend arrays + removedDataSource.close(); + } + catch (Exception e) + { + LOGGER.error("Unable to close datasource ["+ DhSectionPos.toString(removedDataSource.getPos()) +"], error: ["+e.getMessage()+"].", e); + } + + return null; + }); } @@ -146,26 +142,36 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable // List methods // //==============// - public int getUnsavedCount() { return (int)this.dataSourceByPosition.size(); } + public int getUnsavedCount() { return this.dataSourceByPosition.size(); } + public void flush() { this.cleanUp(true); } /** Removes everything from the memory cache and fires the {@link DelayedFullDataSourceSaveCache#onSaveTimeoutAsyncFunc} for each. */ - public void flush() + public void cleanUp(boolean flushAll) { - Set keySet = this.dataSourceByPosition.asMap().keySet(); - for (Long pos : keySet) + Enumeration keyIterator = this.dataSourceByPosition.keys(); + while (keyIterator.hasMoreElements()) { - ReentrantLock lock = this.saveLockContainer.getLockForPos(pos); + Long pos = keyIterator.nextElement(); + ReentrantLock posLock = this.saveLockContainer.getLockForPos(pos); try { - lock.lock(); + posLock.lock(); - this.dataSourceByPosition.invalidate(pos); + DataSourceSavedTimePair savedPair = this.dataSourceByPosition.getOrDefault(pos, null); + if (savedPair != null) + { + if (flushAll + || savedPair.dataSourceHasTimedOut(this.saveDelayInMs)) + { + this.dataSourceByPosition.remove(pos); + this.handleDataSourceRemoval(savedPair.dataSource); + } + } } finally { - lock.unlock(); + posLock.unlock(); } - } } @@ -197,7 +203,7 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable } else { - cache.dataSourceByPosition.cleanUp(); + cache.cleanUp(false); } }); } @@ -240,4 +246,36 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable CompletableFuture saveAsync(FullDataSourceV2 inputDataSource); } + /** + * used to keep track of when data sources + * were written to so we can flush them once + * enough time has passed. + */ + private static class DataSourceSavedTimePair + { + @NotNull + public final FullDataSourceV2 dataSource; + /** the last unix millisecond time this data source was written to */ + public long lastWrittenDateTimeMs; + + + public DataSourceSavedTimePair(@NotNull FullDataSourceV2 dataSource) + { + this.dataSource = dataSource; + this.lastWrittenDateTimeMs = System.currentTimeMillis(); + } + + + public void updateLastWrittenTimestamp() + { this.lastWrittenDateTimeMs = System.currentTimeMillis(); } + + public boolean dataSourceHasTimedOut(long msTillTimeout) + { + long currentTime = System.currentTimeMillis(); + long timeSinceUpdate = currentTime - this.lastWrittenDateTimeMs; + return (timeSinceUpdate > msTillTimeout); + } + } + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java similarity index 86% rename from core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java rename to core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java index 50dca02a7..9ee37500e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListParent.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/AbstractPhantomArrayList.java @@ -16,13 +16,13 @@ import java.lang.ref.PhantomReference; * @see PhantomArrayListCheckout * @see PhantomArrayListPool */ -public abstract class PhantomArrayListParent implements AutoCloseable +public abstract class AbstractPhantomArrayList implements AutoCloseable { private static final Logger LOGGER = LogManager.getLogger(); private final PhantomArrayListPool phantomArrayListPool; - private final PhantomReference phantomReference; + private final PhantomReference phantomReference; /** * It's recommended to set this as null after the child's constructor @@ -37,7 +37,7 @@ public abstract class PhantomArrayListParent implements AutoCloseable //=============// /** The Array counts can be 0 or greater. */ - public PhantomArrayListParent(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount) + public AbstractPhantomArrayList(PhantomArrayListPool phantomArrayListPool, int byteArrayCount, int shortArrayCount, int longArrayCount) { if (byteArrayCount < 0 || shortArrayCount < 0 || longArrayCount < 0) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java index bb9c5a474..95da5674e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListCheckout.java @@ -10,13 +10,12 @@ import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicBoolean; /** * This keeps track of all the poolable * arrays that can be retrieved via the {@link PhantomArrayListPool}. * - * @see PhantomArrayListParent + * @see AbstractPhantomArrayList * @see PhantomArrayListPool */ public class PhantomArrayListCheckout implements AutoCloseable diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java index 42a4d4293..caff83106 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pooling/PhantomArrayListPool.java @@ -33,18 +33,18 @@ import java.util.concurrent.atomic.AtomicInteger; * we pool these arrays when possible.

* * How pooled arrays can be returned:
- * 1. Closing the {@link PhantomArrayListParent}
+ * 1. Closing the {@link AbstractPhantomArrayList}
* The fastest and most efficient method of returning pooled arrays * is to call {@link AutoCloseable#close()}.

* - * 2. {@link PhantomArrayListParent} Garbage Collection
+ * 2. {@link AbstractPhantomArrayList} Garbage Collection
* Some objects are used across many different threads and - * cleanly closing them is impossible, so when the {@link PhantomArrayListParent} + * cleanly closing them is impossible, so when the {@link AbstractPhantomArrayList} * is automatically garbage collected we recover and recycle any * arrays it checked out. * This is less efficient since it may allow a lot of additional arrays to * be created while we wait for the garbage collector to run, but - * does prevent any leaks from {@link PhantomArrayListParent} that weren't closed. + * does prevent any leaks from {@link AbstractPhantomArrayList} that weren't closed. * *

* Use Notes:
@@ -83,9 +83,9 @@ public class PhantomArrayListPool */ public final boolean logGarbageCollectedStacks; - public final ConcurrentHashMap, PhantomArrayListCheckout> + public final ConcurrentHashMap, PhantomArrayListCheckout> phantomRefToCheckout = new ConcurrentHashMap<>(); - public final ReferenceQueue phantomRefQueue = new ReferenceQueue<>(); + public final ReferenceQueue phantomRefQueue = new ReferenceQueue<>(); private final ConcurrentLinkedQueue> pooledCheckoutsRefs = new ConcurrentLinkedQueue<>(); @@ -274,7 +274,7 @@ public class PhantomArrayListPool allocationStackTraceCountPairList.clear(); - Reference phantomRef = pool.phantomRefQueue.poll(); + Reference phantomRef = pool.phantomRefQueue.poll(); while (phantomRef != null) { // return the pooled arrays @@ -378,7 +378,7 @@ public class PhantomArrayListPool // return checkout // //=================// - public void returnParentPhantomRef(@NotNull PhantomReference parentRef) + public void returnParentPhantomRef(@NotNull PhantomReference parentRef) { try { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index a0c80c35e..5a82322bc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -25,7 +25,7 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; +import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.network.INetworkObject; @@ -47,7 +47,7 @@ import java.io.*; /** handles storing {@link FullDataSourceV2}'s in the database. */ public class FullDataSourceV2DTO - extends PhantomArrayListParent + extends AbstractPhantomArrayList implements IBaseDTO, INetworkObject, AutoCloseable { public static final boolean VALIDATE_INPUT_DATAPOINTS = true; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java index dbdd50189..5d8b05571 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointReducingList.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.util; import com.google.common.annotations.VisibleForTesting; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView; -import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent; +import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.util.LodUtil.AssertFailureException; import it.unimi.dsi.fastutil.longs.LongArrayList; @@ -46,7 +46,7 @@ import it.unimi.dsi.fastutil.shorts.ShortArrays; * * @author Builderb0y */ -public class RenderDataPointReducingList extends PhantomArrayListParent +public class RenderDataPointReducingList extends AbstractPhantomArrayList { /** diff --git a/core/src/test/java/tests/DelayedSaveCacheTest.java b/core/src/test/java/tests/DelayedSaveCacheTest.java new file mode 100644 index 000000000..c8b14c0b8 --- /dev/null +++ b/core/src/test/java/tests/DelayedSaveCacheTest.java @@ -0,0 +1,138 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.DelayedFullDataSourceSaveCache; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.KeyedLockContainer; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +/** + * A few very basic tests to confirm {@link DelayedFullDataSourceSaveCache} + * is working properly. + * + * @author James Seibel + * @version 2025-10-02 + */ +public class DelayedSaveCacheTest +{ + + + + @Test + public void CacheExpirationAndPoolingTest() throws InterruptedException + { + // how many times any data source has been "written to disk" + AtomicInteger diskSaveCountRef = new AtomicInteger(0); + + DelayedFullDataSourceSaveCache cache = new DelayedFullDataSourceSaveCache((FullDataSourceV2 fullDataSource) -> + { + diskSaveCountRef.getAndIncrement(); + return this.onDataSourceSaveAsync(fullDataSource); + }, 1_000); + + + + //==============================// + // single item and manual flush // + //==============================// + + PhantomArrayListCheckout initialCheckout; + try (FullDataSourceV2 initialSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte)6, 0, 0))) + { + initialCheckout = initialSource.getPhantomArrayCheckoutForUnitTesting(); + cache.writeDataSourceToMemoryAndQueueSave(initialSource); + } + Assert.assertEquals("only 1 item should be in the cache", 1, cache.getUnsavedCount()); + Assert.assertEquals("no disk saves should have happened yet", 0, diskSaveCountRef.get()); + + // manual flush + cache.flush(); + Assert.assertEquals("memory cache should be empty after", 0, cache.getUnsavedCount()); + Assert.assertEquals("1 manual flush was expected", 1, diskSaveCountRef.get()); + + + + //======================// + // quick group position // + //======================// + + // write multiple items for the same position + for (int i = 0; i < 4; i++) + { + try (FullDataSourceV2 loopSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0))) + { + PhantomArrayListCheckout loopCheckout = loopSource.getPhantomArrayCheckoutForUnitTesting(); + Assert.assertEquals(initialCheckout, loopCheckout); + + cache.writeDataSourceToMemoryAndQueueSave(loopSource); + } + } + // each item writes to the same place + Assert.assertEquals("exactly 1 item should be in the cache", 1, cache.getUnsavedCount()); + Assert.assertEquals("no new saves should have happened yet", 1, diskSaveCountRef.get()); + + // wait for the cache to clear + Thread.sleep(2_000); + Assert.assertEquals("Cache should have automatically cleared due to inactivity", 0, cache.getUnsavedCount()); + Assert.assertEquals("second save after timeout expected", 2, diskSaveCountRef.get()); + + + + //=====================// + // slow group position // + //=====================// + + // write multiple items for the same position + for (int i = 0; i < 4; i++) + { + try (FullDataSourceV2 loopSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0))) + { + PhantomArrayListCheckout loopCheckout = loopSource.getPhantomArrayCheckoutForUnitTesting(); + Assert.assertEquals(initialCheckout, loopCheckout); + + cache.writeDataSourceToMemoryAndQueueSave(loopSource); + } + + // long enough to prevent a timeout, but short enough that they don't happen all at once + Thread.sleep(500); + } + // each item writes to the same place + Assert.assertEquals("exactly 1 item should be in the cache", 1, cache.getUnsavedCount()); + Assert.assertEquals("no new saves should have happened yet", 2, diskSaveCountRef.get()); + + // wait for the cache to clear + Thread.sleep(2_000); + Assert.assertEquals("Cache should have automatically cleared due to inactivity", 0, cache.getUnsavedCount()); + Assert.assertEquals("third timeout expected", 3, diskSaveCountRef.get()); + + } + private CompletableFuture onDataSourceSaveAsync(FullDataSourceV2 fullDataSource) + { return CompletableFuture.completedFuture(null); } + +} diff --git a/core/src/test/java/tests/KeyedLockTest.java b/core/src/test/java/tests/KeyedLockTest.java new file mode 100644 index 000000000..30668f7ce --- /dev/null +++ b/core/src/test/java/tests/KeyedLockTest.java @@ -0,0 +1,65 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.KeyedLockContainer; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.locks.ReentrantLock; + +/** + * @see KeyedLockContainer + * + * @author James Seibel + * @version 2025-10-02 + */ +public class KeyedLockTest +{ + + @Test + public void BasicKeyedLockTest() + { + KeyedLockContainer lockContainer = new KeyedLockContainer<>(); + + for (long a = -10; a < 10; a++) + { + ReentrantLock aLock = lockContainer.getLockForPos(a); + + for (long b = -10; b < 10; b++) + { + ReentrantLock bLock = lockContainer.getLockForPos(a); + + // we only care that the same position always map to the same object + // if different positions map to the same object, + // that's expected hash-collision behavior and is fine + if (a == b) + { + Assert.assertEquals("long values ["+a+"] and ["+b+"] should have returned the same lock", aLock, bLock); + } + } + } + + } + +} diff --git a/core/src/test/java/tests/PooledDataSourceCheckoutTest.java b/core/src/test/java/tests/PooledDataSourceCheckoutTest.java new file mode 100644 index 000000000..215de9721 --- /dev/null +++ b/core/src/test/java/tests/PooledDataSourceCheckoutTest.java @@ -0,0 +1,66 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.DelayedFullDataSourceSaveCache; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.KeyedLockContainer; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @see PhantomArrayListCheckout + * + * @author James Seibel + * @version 2025-10-02 + */ +public class PooledDataSourceCheckoutTest +{ + + @Test + public void TestCheckouts() + { + PhantomArrayListCheckout initialCheckout; + try (FullDataSourceV2 initialSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte)6, 0, 0))) + { + initialCheckout = initialSource.getPhantomArrayCheckoutForUnitTesting(); + } + + try (FullDataSourceV2 outerSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0))) + { + PhantomArrayListCheckout outerCheckout = outerSource.getPhantomArrayCheckoutForUnitTesting(); + Assert.assertEquals("the first checkout object should be pooled", initialCheckout, outerCheckout); + + try (FullDataSourceV2 innerSource = FullDataSourceV2.createEmpty(DhSectionPos.encode((byte) 6, 0, 0))) + { + PhantomArrayListCheckout innerCheckout = innerSource.getPhantomArrayCheckoutForUnitTesting(); + Assert.assertNotEquals("the second checkout object should not be shared when the first is still in use", initialCheckout, innerCheckout); + Assert.assertNotEquals("the second checkout object should not be shared when the first is still in use", outerCheckout, innerCheckout); + } + } + } + +}