diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index cac5aee09..33e89daaa 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -24,7 +24,7 @@ import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -157,7 +157,7 @@ public class FullDataSourceV1 implements IDataSource * Clears and then overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an existing {@link FullDataSourceV1} and can be used in place of a constructor to reuse an existing {@link FullDataSourceV1} object. */ - public void repopulateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { // clear/overwrite the old data this.resizeDataStructuresForRepopulation(dto.pos); @@ -171,7 +171,7 @@ public class FullDataSourceV1 implements IDataSource * Overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an empty {@link FullDataSourceV1} and functions similar to a constructor. */ - public void populateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level); this.setSourceSummaryData(summaryData); @@ -193,13 +193,6 @@ public class FullDataSourceV1 implements IDataSource // low level stream methods // - @Deprecated - @Override - public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException - { - throw new UnsupportedOperationException("Deprecated"); - } - /** unused, just here for reference as to how the data was written */ @Deprecated public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException @@ -210,7 +203,7 @@ public class FullDataSourceV1 implements IDataSource outputStream.writeByte(this.worldGenStep.value); } - public FullDataSourceSummaryData readSourceSummaryInfo(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetail = inputStream.readInt(); if (dataDetail != dto.dataDetailLevel) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index bb86518ff..5a5dff219 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -893,19 +893,6 @@ public class FullDataSourceV2 implements IDataSource - //============// - // deprecated // - //============// - - @Deprecated - @Override - public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) - { - throw new UnsupportedOperationException("deprecated"); - } - - - //=========// // pooling // //=========// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index fd2b8a011..0d40f9e1a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -211,83 +211,9 @@ public class ColumnRenderSource implements IDataSource - //========================// - // data update and output // - //========================// - - @Override - public void writeToStream(DhDataOutputStream outputStream, IDhClientLevel level) throws IOException { this.writeToStream(outputStream); } - public void writeToStream(DhDataOutputStream outputStream) throws IOException - { - outputStream.flush(); - - outputStream.writeByte(this.getDataDetailLevel()); - outputStream.writeInt(this.verticalDataCount); - - if (this.isEmpty) - { - // no data is present - outputStream.writeByte(NO_DATA_FLAG_BYTE); - } - else - { - // data is present - outputStream.writeByte(DATA_GUARD_BYTE); - outputStream.writeInt(this.yOffset); - - // write the data for each column - for (int xz = 0; xz < SECTION_SIZE * SECTION_SIZE; xz++) - { - for (int y = 0; y < this.verticalDataCount; y++) - { - long currentDatapoint = this.renderDataContainer[xz * this.verticalDataCount + y]; - outputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly - } - } - } - - outputStream.writeByte(DATA_GUARD_BYTE); - outputStream.writeByte(this.worldGenStep.value); - - outputStream.flush(); - } - - /** Overrides any data that has not been written directly using write(). Skips empty source dataPoints. */ - public void updateFromRenderSource(ColumnRenderSource renderSource) - { - // validate we are writing for the same location - LodUtil.assertTrue(renderSource.sectionPos.equals(this.sectionPos)); - - // change the vertical size if necessary (this can happen if the vertical quality was changed in the config) - this.clearAndChangeVerticalSize(renderSource.verticalDataCount); - // validate both objects have the same number of dataPoints - LodUtil.assertTrue(renderSource.verticalDataCount == this.verticalDataCount); - - - if (renderSource.isEmpty) - { - // the source is empty, don't attempt to update anything - return; - } - // the source isn't empty, this object won't be empty after the method finishes - this.isEmpty = false; - - localVersion.incrementAndGet(); - } - /** - * If the newVerticalSize is different than the current verticalSize, - * this will delete any data currently in this object and re-size it.
- * Otherwise this method will do nothing. - */ - private void clearAndChangeVerticalSize(int newVerticalSize) - { - if (newVerticalSize != this.verticalDataCount) - { - this.verticalDataCount = newVerticalSize; - this.renderDataContainer = new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]; - this.localVersion.incrementAndGet(); - } - } + //=============// + // data update // + //=============// @Override public boolean update(FullDataSourceV2 inputFullDataSource, IDhClientLevel level) @@ -355,37 +281,12 @@ public class ColumnRenderSource implements IDataSource // data helper methods // //=====================// - public boolean doesDataPointExist(int posX, int posZ) { return RenderDataPointUtil.doesDataPointExist(this.getFirstDataPoint(posX, posZ)); } - - public void generateData(ColumnRenderSource lowerDataContainer, int posX, int posZ) - { - ColumnArrayView outputView = this.getVerticalDataPointView(posX, posZ); - ColumnQuadView quadView = lowerDataContainer.getQuadViewOverRange(posX * 2, posZ * 2, 2, 2); - outputView.mergeMultiDataFrom(quadView); - } - - public int getMaxLodCount() { return SECTION_SIZE * SECTION_SIZE * this.getVerticalSize(); } - - public long getRoughRamUsageInBytes() { return (long) this.renderDataContainer.length * Long.BYTES; } - public DhSectionPos getSectionPos() { return this.sectionPos; } @Override public DhSectionPos getKey() { return this.sectionPos; } public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - /** @return how many data points wide this {@link ColumnRenderSource} is. */ - public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); } - public byte getDetailOffset() { return SECTION_SIZE_OFFSET; } - - public byte getRenderDataFormatVersion() { return DATA_FORMAT_VERSION; } - - /** - * Whether this object is still valid. If not, a new one should be created. - * TODO this will be necessary for dedicated multiplayer support, if the server has newer data this section should no longer be valid - */ - public boolean isValid() { return true; } - public boolean isEmpty() { return this.isEmpty; } public void markNotEmpty() { this.isEmpty = false; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java index 94caa4724..1e816ce0f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; @@ -31,7 +31,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; /** - * Handles loading and parsing {@link LegacyDataSourceDTO}s to create {@link ColumnRenderSource}s.

+ * Handles loading and parsing {@link FullDataSourceV1DTO}s to create {@link ColumnRenderSource}s.

* * Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what * file versions this class can handle. @@ -48,7 +48,7 @@ public class ColumnRenderSourceLoader - public ColumnRenderSource loadRenderSource(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public ColumnRenderSource loadRenderSource(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataFileVersion = dto.binaryDataFormatVersion; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java deleted file mode 100644 index 18b82fe15..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ /dev/null @@ -1,420 +0,0 @@ -package com.seibel.distanthorizons.core.file; - -import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; -import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.TimerUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.channels.ClosedChannelException; -import java.util.Enumeration; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; -import java.util.zip.Adler32; -import java.util.zip.CheckedOutputStream; - -@Deprecated -public abstract class AbstractLegacyDataSourceHandler, TDhLevel extends IDhLevel> - implements AutoCloseable -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); - /** How long a data source must remain un-modified before being written to disk. */ - private static final int SAVE_DELAY_IN_MS = 4_000; - - /** - * The highest numerical detail level known about. - * Used when determining which positions to update. - */ - protected final AtomicInteger topSectionDetailLevelRef; - protected final int minDetailLevel = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - - protected final ConcurrentHashMap unsavedDataSourceBySectionPos = new ConcurrentHashMap<>(); - protected final ConcurrentHashMap saveTimerTasksBySectionPos = new ConcurrentHashMap<>(); - - protected final ReentrantLock[] updateLockArray; - protected final ReentrantLock[] queueSaveLockArray; - protected final ReentrantLock closeLock = new ReentrantLock(); - protected volatile boolean isShutdown = false; - - protected final TDhLevel level; - protected final File saveDir; - - public final AbstractLegacyDataSourceRepo repo; - - - - //=============// - // constructor // - //=============// - - public AbstractLegacyDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } - public AbstractLegacyDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) - { - this.level = level; - this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; - if (!this.saveDir.exists() && !this.saveDir.mkdirs()) - { - LOGGER.warn("Unable to create full data folder, file saving may fail."); - } - - // the lock arrays' length is double the number of CPU cores so the number of collisions - // should be relatively low without having too many extra locks - int lockCount = Runtime.getRuntime().availableProcessors() * 2; - this.updateLockArray = new ReentrantLock[lockCount]; - this.queueSaveLockArray = new ReentrantLock[lockCount]; - for (int i = 0; i < lockCount; i++) - { - this.updateLockArray[i] = new ReentrantLock(); - this.queueSaveLockArray[i] = new ReentrantLock(); - } - - this.repo = this.createRepo(); - - // determine the top detail level currently in the database - int maxSectionDetailLevel = this.repo.getMaxSectionDetailLevel(); - this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel); - } - - - - - //==================// - // abstract methods // - //==================// - - /** When this is called the parent folders should be created */ - protected abstract AbstractLegacyDataSourceRepo createRepo(); - - protected abstract TDataSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException; - /** - * Creates a new data source using any DTOs already present in the database. - * Can return null if there was an issue, but in general should return at least an empty data source. - */ - @Nullable - protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos); - - protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); - - - - //==============// - // data reading // - //==============// - - /** - * Returns the {@link TDataSource} for the given section position.
- * The returned data source may be null if there was a problem.

- * - * This call is concurrent. I.e. it supports being called by multiple threads at the same time. - */ - public CompletableFuture getAsync(DhSectionPos pos) - { - ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return CompletableFuture.completedFuture(null); - } - - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); - } - /** - * Should only be used in internal file handler methods where we are already running on a file handler thread. - * Can return null if there was a problem. - * @see AbstractLegacyDataSourceHandler#getAsync(DhSectionPos) - */ - @Nullable - public TDataSource get(DhSectionPos pos) - { - // used the unsaved data source if present - if (this.unsavedDataSourceBySectionPos.containsKey(pos)) - { - return this.unsavedDataSourceBySectionPos.get(pos); - } - // an unsaved data source isn't present - // check the database - - - // increase the top detail level if necessary - this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - - - TDataSource dataSource = null; - try - { - LegacyDataSourceDTO dto = this.repo.getByKey(pos); - if (dto != null) - { - // load from file - dataSource = this.createDataSourceFromDto(dto); - } - else - { - // attempt to create from any existing files - dataSource = this.createNewDataSourceFromExistingDtos(pos); - } - } - catch (InterruptedException ignore) { } - catch (IOException e) - { - LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); - } - - return dataSource; - } - - - - //===============// - // data updating // - //===============// - - public CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputDataSource) - { - ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return CompletableFuture.completedFuture(null); - } - - - try - { - // run file handling on a separate thread - return CompletableFuture.runAsync(() -> - { - DhSectionPos bottomPos = inputDataSource.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - - bottomPos.forEachPosUpToDetailLevel( - this.topSectionDetailLevelRef.byteValue(), - (pos) -> this.updateDataSourceAtPos(pos, inputDataSource) ); - - }, executor); - } - catch (RejectedExecutionException ignore) - { - // can happen if the executor was shutdown while this task was queued - return CompletableFuture.completedFuture(null); - } - } - protected void updateDataSourceAtPos(DhSectionPos pos, FullDataSourceV2 newDataSource) - { - // a lock is necessary to prevent two threads from writing to the same position at once, - // if that happens only the second update will apply and the LOD will end up with hole(s) - ReentrantLock updateLock = this.getUpdateLockForPos(pos); - - try - { - updateLock.lock(); - - // get or create the data source - TDataSource dataSource = this.get(pos); - if (dataSource == null) - { - dataSource = this.makeEmptyDataSource(pos); - } - dataSource.update(newDataSource, this.level); - - this.queueDelayedSave(dataSource); - } - catch (Exception e) - { - LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); - } - finally - { - updateLock.unlock(); - } - } - /** - * Queues the given data source to save after {@link AbstractLegacyDataSourceHandler#SAVE_DELAY_IN_MS} - * milliseconds have passed without any additional modifications.

- * - * This prevents repeatedly reading/writing the same data source to/from disk if said - * source is currently being updated via world gen or chunk modifications. - * This drastically reduces disk usage and improves performance. - */ - protected void queueDelayedSave(TDataSource dataSource) - { - // a lock is necessary to prevent two threads from queuing a save at the same time, - // which can cause the timer to queue canceled tasks - DhSectionPos pos = dataSource.getSectionPos(); - ReentrantLock saveQueueLock = this.getSaveQueueLockForPos(pos); - - - // done to prevent queueing saves while the current queue is being cleared - if (this.isShutdown) - { - return; - } - - - try - { - saveQueueLock.lock(); - - // put the data source in memory until it can be flushed to disk - this.unsavedDataSourceBySectionPos.put(pos, dataSource); - - TimerTask task = new TimerTask() - { - @Override - public void run() - { - - // remove this task from the queue - AbstractLegacyDataSourceHandler.this.saveTimerTasksBySectionPos.remove(pos); - - try - { - final TDataSource finalDataSource = AbstractLegacyDataSourceHandler.this.unsavedDataSourceBySectionPos.remove(pos); - - // this can rarely happen due to imperfect concurrency handling, - // if the data source is null that just means it has already been saved so nothing needs to be done - if (finalDataSource != null) - { - AbstractLegacyDataSourceHandler.this.writeDataSourceToFile(finalDataSource); - } - } - catch (Exception e) // this can throw errors (not exceptions) when installed in Iris' dev environment for some reason due to an issue with LZ4's compression library - { - LOGGER.error("Failed to save updated data for section ["+pos+"], error: ["+e.getMessage()+"]", e); - } - } - }; - try - { - DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS); - } - catch (IllegalStateException ignore) - { - // James isn't sure why this is possible since this logic is inside a lock, - // maybe the timer is just async enough that there can be problems? - LOGGER.warn("Attempted to queue an already canceled task. Pos: ["+pos+"], task already queued for pos: ["+this.saveTimerTasksBySectionPos.containsKey(pos)+"]"); - } - - - // cancel the old save timer if present - // (this is equivalent to restarting the timer) - TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task); - if (oldTask != null) - { - oldTask.cancel(); - } - } - finally - { - saveQueueLock.unlock(); - } - } - protected void writeDataSourceToFile(TDataSource dataSource) throws IOException - { - LodUtil.assertTrue(dataSource != null); - - try - { - // write the outputs to a stream to prep for writing to the database - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - - // the order of these streams is important, otherwise the checksum won't be calculated - CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); - // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, - // but since this stream will be closed immediately after writing anyway, it won't be an issue - DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut, EDhApiDataCompressionMode.LZ4); - - dataSource.writeToStream(compressedOut, AbstractLegacyDataSourceHandler.this.level); - - compressedOut.flush(); - int checksum = (int) checkedOut.getChecksum().getValue(); - byteArrayOutputStream.close(); - - - // save the DTO - LegacyDataSourceDTO newDto = new LegacyDataSourceDTO( - dataSource.getSectionPos(), checksum, - dataSource.getDataDetailLevel(), EDhApiWorldGenerationStep.EMPTY, ColumnRenderSource.DATA_NAME, - dataSource.getDataFormatVersion(), - byteArrayOutputStream.toByteArray()); - this.repo.save(newDto); - } - catch (ClosedChannelException e) // includes ClosedByInterruptException - { - // expected if the file handler is shut down, the exception can be ignored - } - } - - - - //================// - // helper methods // - //================// - - /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ - protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } - protected ReentrantLock getSaveQueueLockForPos(DhSectionPos pos) { return this.queueSaveLockArray[Math.abs(pos.hashCode()) % this.queueSaveLockArray.length]; } - - - - //=========// - // cleanup // - //=========// - - @Override - public void close() - { - try - { - this.closeLock.lock(); - this.isShutdown = true; - - // wait a moment so any queued saves can finish queuing, - // otherwise we might not see everything that needs saving and attempt to use a closed repo - Thread.sleep(200); - - LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "], saving [" + this.saveTimerTasksBySectionPos.size() + "] positions."); - - - Enumeration list = this.saveTimerTasksBySectionPos.keys(); - while (list.hasMoreElements()) - { - DhSectionPos pos = list.nextElement(); - TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos); - if (saveTask != null) - { - saveTask.run(); - // canceling the task doesn't need to be done since the it has internal logic to prevent running more than once - } - } - - LOGGER.info("[" + this.getClass().getSimpleName() + "] saving complete, closing repo."); - this.repo.close(); - } - catch (InterruptedException ignore) { } - finally - { - this.closeLock.unlock(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 0da2c34b6..edcb69407 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -21,19 +21,9 @@ public interface IDataSource extends IBaseDTO. - */ - package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; -import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; -import com.seibel.distanthorizons.core.sql.repo.LegacyFullDataRepo; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; +import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV1Repo; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -35,34 +15,53 @@ import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.locks.ReentrantLock; -public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler +public class FullDataSourceProviderV1 + implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + protected final ReentrantLock closeLock = new ReentrantLock(); + protected volatile boolean isShutdown = false; + + protected final TDhLevel level; + protected final File saveDir; + + public final FullDataSourceV1Repo repo; + //=============// // constructor // //=============// - public FullDataSourceProviderV1(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + public FullDataSourceProviderV1(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { - super(level, saveStructure, saveDirOverride); + this.level = level; + this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; + if (!this.saveDir.exists() && !this.saveDir.mkdirs()) + { + LOGGER.warn("Unable to create full data folder, file saving may fail."); + } + + this.repo = this.createRepo(); } - //====================// - // Abstract overrides // - //====================// + //==================// + // abstract methods // + //==================// - @Override - protected AbstractLegacyDataSourceRepo createRepo() + /** When this is called the parent folders should be created */ + protected FullDataSourceV1Repo createRepo() { try { - return new LegacyFullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + return new FullDataSourceV1Repo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); } catch (SQLException e) { @@ -72,32 +71,61 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler + * The returned data source may be null if there was a problem.

+ * + * This call is concurrent. I.e. it supports being called by multiple threads at the same time. + */ + public CompletableFuture getAsync(DhSectionPos pos) + { + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + /** + * Should only be used in internal file handler methods where we are already running on a file handler thread. + * Can return null. + * @see FullDataSourceProviderV1#getAsync(DhSectionPos) + */ + @Nullable + public FullDataSourceV1 get(DhSectionPos pos) + { + FullDataSourceV1 dataSource = null; + try + { + FullDataSourceV1DTO dto = this.repo.getByKey(pos); + if (dto != null) + { + // load from file + dataSource = this.createDataSourceFromDto(dto); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); + } + + return dataSource; + } @@ -105,13 +133,13 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler getDataSourcesToMigrate(int limit) { ArrayList dataSourceList = new ArrayList<>(); - ArrayList migrationPosList = ((LegacyFullDataRepo) this.repo).getPositionsToMigrate(limit); + ArrayList migrationPosList = ((FullDataSourceV1Repo) this.repo).getPositionsToMigrate(limit); for (int i = 0; i < migrationPosList.size(); i++) { DhSectionPos pos = migrationPosList.get(i); @@ -125,7 +153,29 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler. - */ - -package com.seibel.distanthorizons.core.file.renderfile; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; - -import java.util.concurrent.CompletableFuture; - -/** - * Handles reading, writing, and updating {@link ColumnRenderSource}'s.
- * Should be backed by a database handled by a {@link RenderDataRepo}. - * - * @deprecated an interface isn't necessary for the single render source provider we have - */ -@Deprecated -public interface IRenderSourceProvider extends AutoCloseable -{ - CompletableFuture getAsync(DhSectionPos pos); - - void updateDataSource(FullDataSourceV2 dataSource); - - /** Deletes any data stored in the render cache so it can be re-created */ - void deleteRenderCache(); - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java deleted file mode 100644 index d0c18963a..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.renderfile; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.logging.f3.F3Screen; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; -import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; -import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.*; - -public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler implements IRenderSourceProvider -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - private final F3Screen.NestedMessage threadPoolMsg; - - public final FullDataSourceProviderV2 fullDataSourceProvider; - - - - //=============// - // constructor // - //=============// - - public RenderSourceFileHandler(FullDataSourceProviderV2 sourceProvider, IDhClientLevel clientLevel, AbstractSaveStructure saveStructure) - { - super(clientLevel, saveStructure); - - this.fullDataSourceProvider = sourceProvider; - this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log); - } - - - - //====================// - // Abstract overrides // - //====================// - - @Override - protected AbstractLegacyDataSourceRepo createRepo() - { - try - { - return new RenderDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); - } - catch (SQLException e) - { - // should only happen if there is an issue with the database (it's locked or can't be created if missing) - // or the database update failed - throw new RuntimeException(e); - } - } - - @Override - protected ColumnRenderSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException - { return ColumnRenderSourceLoader.INSTANCE.loadRenderSource(dto, dto.getInputStream(), this.level); } - @Override - protected ColumnRenderSource createNewDataSourceFromExistingDtos(DhSectionPos pos) - { - ColumnRenderSource renderDataSource; - - try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(pos)) - { - renderDataSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); - } - catch (Exception e) { throw new RuntimeException(e); } - - return renderDataSource; - } - - @Override - protected ColumnRenderSource makeEmptyDataSource(DhSectionPos pos) - { return ColumnRenderSource.createEmptyRenderSource(pos); } - - - - //=====================// - // extension overrides // - //=====================// - - @Override - public void updateDataSource(FullDataSourceV2 inputDataSource) - { - // TODO once the legacy data provider has been replaced this can be removed - this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); - } - - - - //=========// - // F3 menu // - //=========// - - /** Returns what should be displayed in Minecraft's F3 debug menu */ - private String[] f3Log() - { - ThreadPoolExecutor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor(); - String fileQueueSize = (fileExecutor != null) ? fileExecutor.getQueue().size()+"" : "-"; - String fileCompletedTaskSize = (fileExecutor != null) ? fileExecutor.getCompletedTaskCount()+"" : "-"; - - ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor(); - String updateQueueSize = (updateExecutor != null) ? updateExecutor.getQueue().size()+"" : "-"; - String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-"; - - int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount(); - - - - ArrayList lines = new ArrayList<>(); - lines.add("File Handler [" + this.level.getLevelWrapper().getDimensionType().getDimensionName() + "]"); - lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); - lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")"); - lines.add(" Level Unsaved #: " + this.level.getUnsavedDataSourceCount()); - if (unsavedDataSourceCount != -1) - { - lines.add(" File Handler Unsaved #: " + unsavedDataSourceCount); - } - lines.add(" Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size()); - lines.add(" Unsaved render sources: " + this.unsavedDataSourceBySectionPos.size()); - - - - return lines.toArray(new String[0]); - } - - - - //=====================// - // shutdown / clearing // - //=====================// - - public void close() - { - super.close(); - this.threadPoolMsg.close(); - } - - public void deleteRenderCache() { this.repo.deleteAll(); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 7b88e473c..139e2d73f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -26,7 +26,6 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -36,13 +35,16 @@ import com.seibel.distanthorizons.core.render.LodQuadTree; import com.seibel.distanthorizons.core.render.RenderBufferHandler; import com.seibel.distanthorizons.core.render.renderer.LodRenderer; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import org.apache.logging.log4j.Logger; import java.io.Closeable; +import java.util.ArrayList; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc @@ -50,7 +52,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); - private final IDhClientLevel parentClientLevel; + private final IDhClientLevel clientLevel; public final FullDataSourceProviderV2 fullDataSourceProvider; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); @@ -63,12 +65,12 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // constructor // //=============// - public ClientLevelModule(IDhClientLevel parentClientLevel) + public ClientLevelModule(IDhClientLevel clientLevel) { - this.parentClientLevel = parentClientLevel; + this.clientLevel = clientLevel; this.f3Message = new F3Screen.NestedMessage(this::f3Log); - this.fullDataSourceProvider = this.parentClientLevel.getFullDataProvider(); + this.fullDataSourceProvider = this.clientLevel.getFullDataProvider(); this.fullDataSourceProvider.dateSourceUpdateListeners.add(this); } @@ -102,14 +104,14 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle return; } - IClientLevelWrapper clientLevelWrapper = this.parentClientLevel.getClientLevelWrapper(); + IClientLevelWrapper clientLevelWrapper = this.clientLevel.getClientLevelWrapper(); if (clientLevelWrapper == null) { return; } clientRenderState.close(); - clientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure()); + clientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, clientRenderState)) { //FIXME: How to handle this? @@ -141,7 +143,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle /** @return if the {@link ClientRenderState} was successfully swapped */ public boolean startRenderer(IClientLevelWrapper clientLevelWrapper) { - ClientRenderState ClientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure()); + // TODO why are we passing in a level wrapper? Our client level is already defined. + ClientRenderState ClientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, ClientRenderState)) { LOGGER.warn("Failed to start renderer due to concurrency"); @@ -208,15 +211,14 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // data handling // //===============// - public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } + public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientLevel.getFullDataProvider().updateDataSourceAsync(data); } @Override public void OnDataSourceUpdated(FullDataSourceV2 updatedFullDataSource) { - // if rendering also update the render sources + // if rendering, also update the render sources ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { - ClientRenderState.renderSourceFileHandler.updateDataSource(updatedFullDataSource); ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos()); } } @@ -254,19 +256,40 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // misc helper functions // //=======================// - /** Returns what should be displayed in Minecraft's F3 debug menu */ - protected String[] f3Log() + private String[] f3Log() { - String dimName = this.parentClientLevel.getClientLevelWrapper().getDimensionType().getDimensionName(); - ClientRenderState renderState = this.ClientRenderStateRef.get(); - if (renderState == null) + String dimName = this.clientLevel.getLevelWrapper().getDimensionType().getDimensionName(); + boolean rendererActive = this.ClientRenderStateRef.get() != null; + + ThreadPoolExecutor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor(); + String fileQueueSize = (fileExecutor != null) ? fileExecutor.getQueue().size()+"" : "-"; + String fileCompletedTaskSize = (fileExecutor != null) ? fileExecutor.getCompletedTaskCount()+"" : "-"; + + ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor(); + String updateQueueSize = (updateExecutor != null) ? updateExecutor.getQueue().size()+"" : "-"; + String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-"; + + int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount(); + + + ArrayList lines = new ArrayList<>(); + lines.add(""); + lines.add("level [" + dimName + "] rendering: " + (rendererActive ? "Active" : "Inactive")); + // TODO a lot of these items only need to be rendered once, but we don't currently have a way of doing that, so only add them for the rendered level + if (rendererActive) { - return new String[]{"level @ " + dimName + ": Inactive"}; - } - else - { - return new String[]{"level @ " + dimName + ": Active"}; + lines.add("File Handler [" + dimName + "]"); + lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); + lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")"); + lines.add(" Level Unsaved #: " + this.clientLevel.getUnsavedDataSourceCount()); + if (unsavedDataSourceCount != -1) + { + lines.add(" File Handler Unsaved #: " + unsavedDataSourceCount); + } + lines.add(" Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size()); } + + return lines.toArray(new String[0]); } public void clearRenderCache() @@ -299,7 +322,6 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle public final IClientLevelWrapper clientLevelWrapper; public final LodQuadTree quadtree; - public final RenderSourceFileHandler renderSourceFileHandler; public final LodRenderer renderer; public ClientRenderState( @@ -307,12 +329,11 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle AbstractSaveStructure saveStructure) { this.clientLevelWrapper = clientLevelWrapper; - this.renderSourceFileHandler = new RenderSourceFileHandler(fullDataSourceProvider, dhClientLevel, saveStructure); this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get() * LodUtil.CHUNK_WIDTH * 2, // initial position is (0,0) just in case the player hasn't loaded in yet, the tree will be moved once the level starts ticking 0, 0, - this.renderSourceFileHandler.fullDataSourceProvider, this.renderSourceFileHandler); + fullDataSourceProvider); RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree); this.renderer = new LodRenderer(renderBufferHandler); @@ -326,7 +347,6 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle this.renderer.close(); this.quadtree.close(); - this.renderSourceFileHandler.close(); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index e2f512aa5..ac69e5126 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -24,7 +24,6 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; @@ -60,7 +59,6 @@ public class LodQuadTree extends QuadTree implements AutoClose public final int blockRenderDistanceDiameter; private final FullDataSourceProviderV2 fullDataSourceProvider; - private final IRenderSourceProvider renderSourceProvider; /** * This holds every {@link DhSectionPos} that should be reloaded next tick.
@@ -91,14 +89,12 @@ public class LodQuadTree extends QuadTree implements AutoClose public LodQuadTree( IDhClientLevel level, int viewDiameterInBlocks, int initialPlayerBlockX, int initialPlayerBlockZ, - FullDataSourceProviderV2 fullDataSourceProvider, - IRenderSourceProvider renderSourceProvider) + FullDataSourceProviderV2 fullDataSourceProvider) { super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL); this.level = level; this.fullDataSourceProvider = fullDataSourceProvider; - this.renderSourceProvider = renderSourceProvider; this.blockRenderDistanceDiameter = viewDiameterInBlocks; this.horizontalScaleChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, (newHorizontalScale) -> this.onHorizontalQualityChange()); @@ -164,7 +160,7 @@ public class LodQuadTree extends QuadTree implements AutoClose LodRenderSection renderSection = this.getValue(pos); if (renderSection != null) { - renderSection.reload(this.renderSourceProvider); + renderSection.reload(this.fullDataSourceProvider); } } catch (IndexOutOfBoundsException e) @@ -313,7 +309,7 @@ public class LodQuadTree extends QuadTree implements AutoClose { // prepare this section for rendering // TODO this should fire for the lowest detail level first to improve loading speed - renderSection.loadRenderSource(this.renderSourceProvider, this.level); + renderSection.loadRenderSource(this.fullDataSourceProvider, this.level); // wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections if (!parentRenderSectionIsEnabled && renderSection.canRenderNow()) @@ -435,10 +431,7 @@ public class LodQuadTree extends QuadTree implements AutoClose { try { - LOGGER.info("Clearing render cache..."); - - // delete the cache first so the nodes won't accidentally try re-loading the old data - this.renderSourceProvider.deleteRenderCache(); + LOGGER.info("Disposing render data..."); // clear the tree Iterator> nodeIterator = this.nodeIterator(); @@ -452,7 +445,7 @@ public class LodQuadTree extends QuadTree implements AutoClose } } - LOGGER.info("Render cache invalidated, please wait a moment for everything to reload..."); + LOGGER.info("Render data cleared, please wait a moment for everything to reload..."); } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 9063ec741..e70f20e94 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -20,11 +20,12 @@ package com.seibel.distanthorizons.core.render; import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; +import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -57,13 +58,13 @@ public class LodRenderSection implements IDebugRenderable private boolean isRenderingEnabled = false; /** - * If this is true, then {@link LodRenderSection#reload(IRenderSourceProvider)} was called while - * a {@link IRenderSourceProvider} was already being loaded. + * If this is true, then {@ link LodRenderSection#reload(IRenderSourceProvider)} was called while + * a {@ link IRenderSourceProvider} was already being loaded. */ private boolean reloadRenderSourceOnceLoaded = false; - private IRenderSourceProvider renderSourceProvider = null; - private CompletableFuture renderSourceLoadFuture; + private FullDataSourceProviderV2 fullDataSourceProvider = null; + private CompletableFuture fullDataSourceLoadFuture; private ColumnRenderSource renderSource; private IDhClientLevel level = null; @@ -120,17 +121,17 @@ public class LodRenderSection implements IDebugRenderable //=============// /** does nothing if a render source is already loaded or in the process of loading */ - public void loadRenderSource(IRenderSourceProvider renderDataProvider, IDhClientLevel level) + public void loadRenderSource(FullDataSourceProviderV2 fullDataSourceProvider, IDhClientLevel level) { - this.renderSourceProvider = renderDataProvider; + this.fullDataSourceProvider = fullDataSourceProvider; this.level = level; - if (this.renderSourceProvider == null) + if (this.fullDataSourceProvider == null) { LOGGER.warn("LodRenderSection [" + this.pos + "] called loadRenderSource with a empty source provider"); return; } // don't re-load or double load the render source - if (this.renderSource != null || this.renderSourceLoadFuture != null) + if (this.renderSource != null || this.fullDataSourceLoadFuture != null) { // since the render source has been loaded, make sure the render buffers are populated // FIXME this is a duck tape solution, since the renderBufferRef should be populated elsewhere, but this does fix empty LODs when moving around the world @@ -145,7 +146,7 @@ public class LodRenderSection implements IDebugRenderable this.startLoadRenderSourceAsync(); } - public void reload(IRenderSourceProvider renderDataProvider) + public void reload(FullDataSourceProviderV2 fullDataSourceProvider) { // debug rendering boolean showRenderSectionStatus = Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus.get(); @@ -160,8 +161,8 @@ public class LodRenderSection implements IDebugRenderable } - this.renderSourceProvider = renderDataProvider; - if (this.renderSourceProvider == null) + this.fullDataSourceProvider = fullDataSourceProvider; + if (this.fullDataSourceProvider == null) { LOGGER.warn("LodRenderSection [" + this.pos + "] called reload with a empty source provider"); return; @@ -173,7 +174,7 @@ public class LodRenderSection implements IDebugRenderable return; } // wait for the current load future to finish before re-loading - if (this.renderSourceLoadFuture != null) + if (this.fullDataSourceLoadFuture != null) { this.reloadRenderSourceOnceLoaded = true; return; @@ -184,19 +185,22 @@ public class LodRenderSection implements IDebugRenderable private void startLoadRenderSourceAsync() { - this.renderSourceLoadFuture = this.renderSourceProvider.getAsync(this.pos); - this.renderSourceLoadFuture.whenComplete((renderSource, ex) -> + this.fullDataSourceLoadFuture = this.fullDataSourceProvider.getAsync(this.pos); + this.fullDataSourceLoadFuture.whenComplete((fullDataSource, ex) -> { - this.renderSource = renderSource; + // this runs on the a file handler thread, so transforming the data + // here shouldn't cause any stutters + // (Although it might be good to have it on a separate thread anyway) + this.renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); this.lastNs = -1; this.markBufferDirty(); if (this.reloadRenderSourceOnceLoaded) { this.reloadRenderSourceOnceLoaded = false; - this.reload(this.renderSourceProvider); + this.reload(this.fullDataSourceProvider); } - this.renderSourceLoadFuture = null; + this.fullDataSourceLoadFuture = null; }); } @@ -213,7 +217,7 @@ public class LodRenderSection implements IDebugRenderable public boolean canRenderNow() { - if (this.renderSourceLoadFuture != null || this.buildRenderBufferFuture != null) + if (this.fullDataSourceLoadFuture != null || this.buildRenderBufferFuture != null) { // wait for loading to finish return false; @@ -278,7 +282,7 @@ public class LodRenderSection implements IDebugRenderable } /** @return true if this section is loaded and set to render */ - public boolean canBuildBuffer() { return this.renderSourceLoadFuture == null && this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); } + public boolean canBuildBuffer() { return this.fullDataSourceLoadFuture == null && this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); } private boolean isBufferOutdated() { return this.neighborUpdated || this.renderSource.localVersion.get() != this.lastSwapLocalVersion; } /** @return true if this section is loaded and set to render */ @@ -296,10 +300,10 @@ public class LodRenderSection implements IDebugRenderable this.disposeActiveBuffer = true; this.renderSource = null; - if (this.renderSourceLoadFuture != null) + if (this.fullDataSourceLoadFuture != null) { - this.renderSourceLoadFuture.cancel(true); - this.renderSourceLoadFuture = null; + this.fullDataSourceLoadFuture.cancel(true); + this.fullDataSourceLoadFuture = null; } } @@ -473,7 +477,7 @@ public class LodRenderSection implements IDebugRenderable return "LodRenderSection{" + "pos=" + this.pos + ", lodRenderSource=" + this.renderSource + - ", loadFuture=" + this.renderSourceLoadFuture + + ", loadFuture=" + this.fullDataSourceLoadFuture + ", isRenderEnabled=" + this.isRenderingEnabled + '}'; } @@ -496,11 +500,11 @@ public class LodRenderSection implements IDebugRenderable public void debugRender(DebugRenderer debugRenderer) { Color color = Color.red; - if (this.renderSourceProvider == null) + if (this.fullDataSourceProvider == null) { color = Color.black; } - else if (this.renderSourceLoadFuture != null) + else if (this.fullDataSourceLoadFuture != null) { color = Color.yellow; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java similarity index 81% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java index e89a3c7c7..6c5d1d7ce 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java @@ -32,20 +32,12 @@ import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; /** - * Handles storing {@link ColumnRenderSource}'s and {@link FullDataSourceV1}'s in the database. - * - * @deprecated {@link ColumnRenderSource} should store the actual GPU buffer data, - * at that point this DTO should be just used for {@link FullDataSourceV1} - * and could be renamed to FullDataSourceV1DTO + * Handles storing{@link FullDataSourceV1}'s in the database. */ -@Deprecated -public class LegacyDataSourceDTO implements IBaseDTO +public class FullDataSourceV1DTO implements IBaseDTO { public DhSectionPos pos; public int checksum; - /** @deprecated the database now has a last modified date time that should be used instead */ - @Deprecated - public AtomicLong dataVersion = new AtomicLong(Long.MAX_VALUE); public byte dataDetailLevel; public EDhApiWorldGenerationStep worldGenStep; @@ -62,7 +54,7 @@ public class LegacyDataSourceDTO implements IBaseDTO // constructor // //=============// - public LegacyDataSourceDTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) + public FullDataSourceV1DTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) { this.pos = pos; this.checksum = checksum; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java similarity index 64% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index 32d67d19b..521163695 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -21,34 +21,55 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo +public class FullDataSourceV1Repo extends AbstractDhRepo { - public AbstractLegacyDataSourceRepo(String databaseType, String databaseLocation) throws SQLException + public static final String TABLE_NAME = "Legacy_FullData_V1"; + + + + //=============// + // constructor // + //=============// + + public FullDataSourceV1Repo(String databaseType, String databaseLocation) throws SQLException { - super(databaseType, databaseLocation, LegacyDataSourceDTO.class); + super(databaseType, databaseLocation, FullDataSourceV1DTO.class); } + //===========// + // overrides // + //===========// + + @Override + public String getTableName() { return TABLE_NAME; } + + @Override + public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } + + + //=======================// // repo required methods // //=======================// @Override - public LegacyDataSourceDTO convertDictionaryToDto(Map objectMap) throws ClassCastException + public FullDataSourceV1DTO convertDictionaryToDto(Map objectMap) throws ClassCastException { String posString = (String) objectMap.get("DhSectionPos"); DhSectionPos pos = DhSectionPos.deserialize(posString); // meta data int checksum = (Integer) objectMap.get("Checksum"); - long dataVersion = (Long) objectMap.get("DataVersion"); byte dataDetailLevel = (Byte) objectMap.get("DataDetailLevel"); String worldGenStepString = (String) objectMap.get("WorldGenStep"); EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromName(worldGenStepString); @@ -59,7 +80,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( + "select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1"); + + if (resultMap == null) + { + return 0; + } + else + { + int count = (int) resultMap.get("itemCount"); + return count; + } + } + + /** Returns the new "returnCount" positions that need to be migrated */ + public ArrayList getPositionsToMigrate(int returnCount) + { + ArrayList list = new ArrayList<>(); + + List> resultMapList = this.queryDictionary( + "select DhSectionPos " + + "from "+this.getTableName()+" " + + "WHERE MigrationFailed <> 1 " + + "LIMIT "+returnCount+";"); + + for (Map resultMap : resultMapList) + { + // returned in the format [sectionDetailLevel,x,z] IE [6,0,0] + DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos")); + list.add(sectionPos); + } + + return list; + } + + public void markMigrationFailed(DhSectionPos pos) + { + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET MigrationFailed = 1 \n" + + "WHERE DhSectionPos = '"+pos.serialize()+"'"; + + this.queryDictionaryFirst(sql); + } + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java deleted file mode 100644 index 2d191da4e..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.sql.repo; - -import com.seibel.distanthorizons.core.pos.DhSectionPos; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo -{ - public static final String TABLE_NAME = "Legacy_FullData_V1"; - - - public LegacyFullDataRepo(String databaseType, String databaseLocation) throws SQLException - { - super(databaseType, databaseLocation); - } - - - @Override - public String getTableName() { return TABLE_NAME; } - - @Override - public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } - - - - //===========// - // migration // - //===========// - - /** Returns how many positions need to be migrated over to the new version */ - public int getMigrationCount() - { - Map resultMap = this.queryDictionaryFirst( - "select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1"); - - if (resultMap == null) - { - return 0; - } - else - { - int count = (int) resultMap.get("itemCount"); - return count; - } - } - - /** Returns the new "returnCount" positions that need to be migrated */ - public ArrayList getPositionsToMigrate(int returnCount) - { - ArrayList list = new ArrayList<>(); - - List> resultMapList = this.queryDictionary( - "select DhSectionPos " + - "from "+this.getTableName()+" " + - "WHERE MigrationFailed <> 1 " + - "LIMIT "+returnCount+";"); - - for (Map resultMap : resultMapList) - { - // returned in the format [sectionDetailLevel,x,z] IE [6,0,0] - DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos")); - list.add(sectionPos); - } - - return list; - } - - public void markMigrationFailed(DhSectionPos pos) - { - String sql = - "UPDATE "+this.getTableName()+" \n" + - "SET MigrationFailed = 1 \n" + - "WHERE DhSectionPos = '"+pos.serialize()+"'"; - - this.queryDictionaryFirst(sql); - } - - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java deleted file mode 100644 index fa43e81ea..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.sql.repo; - -import com.seibel.distanthorizons.core.pos.DhSectionPos; - -import java.sql.SQLException; - -public class RenderDataRepo extends AbstractLegacyDataSourceRepo -{ - public static final String TABLE_NAME = "DhRenderData"; - - - public RenderDataRepo(String databaseType, String databaseLocation) throws SQLException - { - super(databaseType, databaseLocation); - } - - - @Override - public String getTableName() { return TABLE_NAME; } - - @Override - public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } - -}