Remove render data file handling and related code

This commit is contained in:
James Seibel
2024-03-23 17:55:59 -05:00
parent b64df318ce
commit 8c5cd7f11f
16 changed files with 281 additions and 1059 deletions
@@ -24,7 +24,7 @@ import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import com.seibel.distanthorizons.core.util.FullDataPointUtilV1;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
@@ -157,7 +157,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
* Clears and then overwrites any data in this object with the data from the given file and stream.
* This is expected to be used with an existing {@link FullDataSourceV1} and can be used in place of a constructor to reuse an existing {@link FullDataSourceV1} object.
*/
public void repopulateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
// clear/overwrite the old data
this.resizeDataStructuresForRepopulation(dto.pos);
@@ -171,7 +171,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
* Overwrites any data in this object with the data from the given file and stream.
* This is expected to be used with an empty {@link FullDataSourceV1} and functions similar to a constructor.
*/
public void populateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException
{
FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level);
this.setSourceSummaryData(summaryData);
@@ -193,13 +193,6 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
// low level stream methods //
@Deprecated
@Override
public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException
{
throw new UnsupportedOperationException("Deprecated");
}
/** unused, just here for reference as to how the data was written */
@Deprecated
public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException
@@ -210,7 +203,7 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
outputStream.writeByte(this.worldGenStep.value);
}
public FullDataSourceSummaryData readSourceSummaryInfo(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
public FullDataSourceSummaryData readSourceSummaryInfo(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataDetail = inputStream.readInt();
if (dataDetail != dto.dataDetailLevel)
@@ -893,19 +893,6 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
//============//
// deprecated //
//============//
@Deprecated
@Override
public void writeToStream(DhDataOutputStream outputStream, IDhLevel level)
{
throw new UnsupportedOperationException("deprecated");
}
//=========//
// pooling //
//=========//
@@ -211,83 +211,9 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
//========================//
// data update and output //
//========================//
@Override
public void writeToStream(DhDataOutputStream outputStream, IDhClientLevel level) throws IOException { this.writeToStream(outputStream); }
public void writeToStream(DhDataOutputStream outputStream) throws IOException
{
outputStream.flush();
outputStream.writeByte(this.getDataDetailLevel());
outputStream.writeInt(this.verticalDataCount);
if (this.isEmpty)
{
// no data is present
outputStream.writeByte(NO_DATA_FLAG_BYTE);
}
else
{
// data is present
outputStream.writeByte(DATA_GUARD_BYTE);
outputStream.writeInt(this.yOffset);
// write the data for each column
for (int xz = 0; xz < SECTION_SIZE * SECTION_SIZE; xz++)
{
for (int y = 0; y < this.verticalDataCount; y++)
{
long currentDatapoint = this.renderDataContainer[xz * this.verticalDataCount + y];
outputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly
}
}
}
outputStream.writeByte(DATA_GUARD_BYTE);
outputStream.writeByte(this.worldGenStep.value);
outputStream.flush();
}
/** Overrides any data that has not been written directly using write(). Skips empty source dataPoints. */
public void updateFromRenderSource(ColumnRenderSource renderSource)
{
// validate we are writing for the same location
LodUtil.assertTrue(renderSource.sectionPos.equals(this.sectionPos));
// change the vertical size if necessary (this can happen if the vertical quality was changed in the config)
this.clearAndChangeVerticalSize(renderSource.verticalDataCount);
// validate both objects have the same number of dataPoints
LodUtil.assertTrue(renderSource.verticalDataCount == this.verticalDataCount);
if (renderSource.isEmpty)
{
// the source is empty, don't attempt to update anything
return;
}
// the source isn't empty, this object won't be empty after the method finishes
this.isEmpty = false;
localVersion.incrementAndGet();
}
/**
* If the newVerticalSize is different than the current verticalSize,
* this will delete any data currently in this object and re-size it. <Br>
* Otherwise this method will do nothing.
*/
private void clearAndChangeVerticalSize(int newVerticalSize)
{
if (newVerticalSize != this.verticalDataCount)
{
this.verticalDataCount = newVerticalSize;
this.renderDataContainer = new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount];
this.localVersion.incrementAndGet();
}
}
//=============//
// data update //
//=============//
@Override
public boolean update(FullDataSourceV2 inputFullDataSource, IDhClientLevel level)
@@ -355,37 +281,12 @@ public class ColumnRenderSource implements IDataSource<IDhClientLevel>
// data helper methods //
//=====================//
public boolean doesDataPointExist(int posX, int posZ) { return RenderDataPointUtil.doesDataPointExist(this.getFirstDataPoint(posX, posZ)); }
public void generateData(ColumnRenderSource lowerDataContainer, int posX, int posZ)
{
ColumnArrayView outputView = this.getVerticalDataPointView(posX, posZ);
ColumnQuadView quadView = lowerDataContainer.getQuadViewOverRange(posX * 2, posZ * 2, 2, 2);
outputView.mergeMultiDataFrom(quadView);
}
public int getMaxLodCount() { return SECTION_SIZE * SECTION_SIZE * this.getVerticalSize(); }
public long getRoughRamUsageInBytes() { return (long) this.renderDataContainer.length * Long.BYTES; }
public DhSectionPos getSectionPos() { return this.sectionPos; }
@Override
public DhSectionPos getKey() { return this.sectionPos; }
public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); }
/** @return how many data points wide this {@link ColumnRenderSource} is. */
public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); }
public byte getDetailOffset() { return SECTION_SIZE_OFFSET; }
public byte getRenderDataFormatVersion() { return DATA_FORMAT_VERSION; }
/**
* Whether this object is still valid. If not, a new one should be created.
* TODO this will be necessary for dedicated multiplayer support, if the server has newer data this section should no longer be valid
*/
public boolean isValid() { return true; }
public boolean isEmpty() { return this.isEmpty; }
public void markNotEmpty() { this.isEmpty = false; }
@@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.render;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import org.apache.logging.log4j.Logger;
@@ -31,7 +31,7 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Handles loading and parsing {@link LegacyDataSourceDTO}s to create {@link ColumnRenderSource}s. <br><br>
* Handles loading and parsing {@link FullDataSourceV1DTO}s to create {@link ColumnRenderSource}s. <br><br>
*
* Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what
* file versions this class can handle.
@@ -48,7 +48,7 @@ public class ColumnRenderSourceLoader
public ColumnRenderSource loadRenderSource(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
public ColumnRenderSource loadRenderSource(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException
{
int dataFileVersion = dto.binaryDataFormatVersion;
@@ -1,420 +0,0 @@
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO;
import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.TimerUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.channels.ClosedChannelException;
import java.util.Enumeration;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.Adler32;
import java.util.zip.CheckedOutputStream;
@Deprecated
public abstract class AbstractLegacyDataSourceHandler<TDataSource
extends IDataSource<TDhLevel>, TDhLevel extends IDhLevel>
implements AutoCloseable
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer");
/** How long a data source must remain un-modified before being written to disk. */
private static final int SAVE_DELAY_IN_MS = 4_000;
/**
* The highest numerical detail level known about.
* Used when determining which positions to update.
*/
protected final AtomicInteger topSectionDetailLevelRef;
protected final int minDetailLevel = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
protected final ConcurrentHashMap<DhSectionPos, TDataSource> unsavedDataSourceBySectionPos = new ConcurrentHashMap<>();
protected final ConcurrentHashMap<DhSectionPos, TimerTask> saveTimerTasksBySectionPos = new ConcurrentHashMap<>();
protected final ReentrantLock[] updateLockArray;
protected final ReentrantLock[] queueSaveLockArray;
protected final ReentrantLock closeLock = new ReentrantLock();
protected volatile boolean isShutdown = false;
protected final TDhLevel level;
protected final File saveDir;
public final AbstractLegacyDataSourceRepo repo;
//=============//
// constructor //
//=============//
public AbstractLegacyDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); }
public AbstractLegacyDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride)
{
this.level = level;
this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride;
if (!this.saveDir.exists() && !this.saveDir.mkdirs())
{
LOGGER.warn("Unable to create full data folder, file saving may fail.");
}
// the lock arrays' length is double the number of CPU cores so the number of collisions
// should be relatively low without having too many extra locks
int lockCount = Runtime.getRuntime().availableProcessors() * 2;
this.updateLockArray = new ReentrantLock[lockCount];
this.queueSaveLockArray = new ReentrantLock[lockCount];
for (int i = 0; i < lockCount; i++)
{
this.updateLockArray[i] = new ReentrantLock();
this.queueSaveLockArray[i] = new ReentrantLock();
}
this.repo = this.createRepo();
// determine the top detail level currently in the database
int maxSectionDetailLevel = this.repo.getMaxSectionDetailLevel();
this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel);
}
//==================//
// abstract methods //
//==================//
/** When this is called the parent folders should be created */
protected abstract AbstractLegacyDataSourceRepo createRepo();
protected abstract TDataSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException;
/**
* Creates a new data source using any DTOs already present in the database.
* Can return null if there was an issue, but in general should return at least an empty data source.
*/
@Nullable
protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos);
protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos);
//==============//
// data reading //
//==============//
/**
* Returns the {@link TDataSource} for the given section position. <Br>
* The returned data source may be null if there was a problem. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
public CompletableFuture<TDataSource> getAsync(DhSectionPos pos)
{
ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
return CompletableFuture.supplyAsync(() -> this.get(pos), executor);
}
/**
* Should only be used in internal file handler methods where we are already running on a file handler thread.
* Can return null if there was a problem.
* @see AbstractLegacyDataSourceHandler#getAsync(DhSectionPos)
*/
@Nullable
public TDataSource get(DhSectionPos pos)
{
// used the unsaved data source if present
if (this.unsavedDataSourceBySectionPos.containsKey(pos))
{
return this.unsavedDataSourceBySectionPos.get(pos);
}
// an unsaved data source isn't present
// check the database
// increase the top detail level if necessary
this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel()));
TDataSource dataSource = null;
try
{
LegacyDataSourceDTO dto = this.repo.getByKey(pos);
if (dto != null)
{
// load from file
dataSource = this.createDataSourceFromDto(dto);
}
else
{
// attempt to create from any existing files
dataSource = this.createNewDataSourceFromExistingDtos(pos);
}
}
catch (InterruptedException ignore) { }
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e);
}
return dataSource;
}
//===============//
// data updating //
//===============//
public CompletableFuture<Void> updateDataSourceAsync(FullDataSourceV2 inputDataSource)
{
ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
try
{
// run file handling on a separate thread
return CompletableFuture.runAsync(() ->
{
DhSectionPos bottomPos = inputDataSource.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
bottomPos.forEachPosUpToDetailLevel(
this.topSectionDetailLevelRef.byteValue(),
(pos) -> this.updateDataSourceAtPos(pos, inputDataSource) );
}, executor);
}
catch (RejectedExecutionException ignore)
{
// can happen if the executor was shutdown while this task was queued
return CompletableFuture.completedFuture(null);
}
}
protected void updateDataSourceAtPos(DhSectionPos pos, FullDataSourceV2 newDataSource)
{
// a lock is necessary to prevent two threads from writing to the same position at once,
// if that happens only the second update will apply and the LOD will end up with hole(s)
ReentrantLock updateLock = this.getUpdateLockForPos(pos);
try
{
updateLock.lock();
// get or create the data source
TDataSource dataSource = this.get(pos);
if (dataSource == null)
{
dataSource = this.makeEmptyDataSource(pos);
}
dataSource.update(newDataSource, this.level);
this.queueDelayedSave(dataSource);
}
catch (Exception e)
{
LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e);
}
finally
{
updateLock.unlock();
}
}
/**
* Queues the given data source to save after {@link AbstractLegacyDataSourceHandler#SAVE_DELAY_IN_MS}
* milliseconds have passed without any additional modifications. <br> <br>
*
* This prevents repeatedly reading/writing the same data source to/from disk if said
* source is currently being updated via world gen or chunk modifications.
* This drastically reduces disk usage and improves performance.
*/
protected void queueDelayedSave(TDataSource dataSource)
{
// a lock is necessary to prevent two threads from queuing a save at the same time,
// which can cause the timer to queue canceled tasks
DhSectionPos pos = dataSource.getSectionPos();
ReentrantLock saveQueueLock = this.getSaveQueueLockForPos(pos);
// done to prevent queueing saves while the current queue is being cleared
if (this.isShutdown)
{
return;
}
try
{
saveQueueLock.lock();
// put the data source in memory until it can be flushed to disk
this.unsavedDataSourceBySectionPos.put(pos, dataSource);
TimerTask task = new TimerTask()
{
@Override
public void run()
{
// remove this task from the queue
AbstractLegacyDataSourceHandler.this.saveTimerTasksBySectionPos.remove(pos);
try
{
final TDataSource finalDataSource = AbstractLegacyDataSourceHandler.this.unsavedDataSourceBySectionPos.remove(pos);
// this can rarely happen due to imperfect concurrency handling,
// if the data source is null that just means it has already been saved so nothing needs to be done
if (finalDataSource != null)
{
AbstractLegacyDataSourceHandler.this.writeDataSourceToFile(finalDataSource);
}
}
catch (Exception e) // this can throw errors (not exceptions) when installed in Iris' dev environment for some reason due to an issue with LZ4's compression library
{
LOGGER.error("Failed to save updated data for section ["+pos+"], error: ["+e.getMessage()+"]", e);
}
}
};
try
{
DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS);
}
catch (IllegalStateException ignore)
{
// James isn't sure why this is possible since this logic is inside a lock,
// maybe the timer is just async enough that there can be problems?
LOGGER.warn("Attempted to queue an already canceled task. Pos: ["+pos+"], task already queued for pos: ["+this.saveTimerTasksBySectionPos.containsKey(pos)+"]");
}
// cancel the old save timer if present
// (this is equivalent to restarting the timer)
TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task);
if (oldTask != null)
{
oldTask.cancel();
}
}
finally
{
saveQueueLock.unlock();
}
}
protected void writeDataSourceToFile(TDataSource dataSource) throws IOException
{
LodUtil.assertTrue(dataSource != null);
try
{
// write the outputs to a stream to prep for writing to the database
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// the order of these streams is important, otherwise the checksum won't be calculated
CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32());
// normally a DhStream should be the topmost stream to prevent closing the stream accidentally,
// but since this stream will be closed immediately after writing anyway, it won't be an issue
DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut, EDhApiDataCompressionMode.LZ4);
dataSource.writeToStream(compressedOut, AbstractLegacyDataSourceHandler.this.level);
compressedOut.flush();
int checksum = (int) checkedOut.getChecksum().getValue();
byteArrayOutputStream.close();
// save the DTO
LegacyDataSourceDTO newDto = new LegacyDataSourceDTO(
dataSource.getSectionPos(), checksum,
dataSource.getDataDetailLevel(), EDhApiWorldGenerationStep.EMPTY, ColumnRenderSource.DATA_NAME,
dataSource.getDataFormatVersion(),
byteArrayOutputStream.toByteArray());
this.repo.save(newDto);
}
catch (ClosedChannelException e) // includes ClosedByInterruptException
{
// expected if the file handler is shut down, the exception can be ignored
}
}
//================//
// helper methods //
//================//
/** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */
protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; }
protected ReentrantLock getSaveQueueLockForPos(DhSectionPos pos) { return this.queueSaveLockArray[Math.abs(pos.hashCode()) % this.queueSaveLockArray.length]; }
//=========//
// cleanup //
//=========//
@Override
public void close()
{
try
{
this.closeLock.lock();
this.isShutdown = true;
// wait a moment so any queued saves can finish queuing,
// otherwise we might not see everything that needs saving and attempt to use a closed repo
Thread.sleep(200);
LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "], saving [" + this.saveTimerTasksBySectionPos.size() + "] positions.");
Enumeration<DhSectionPos> list = this.saveTimerTasksBySectionPos.keys();
while (list.hasMoreElements())
{
DhSectionPos pos = list.nextElement();
TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos);
if (saveTask != null)
{
saveTask.run();
// canceling the task doesn't need to be done since the it has internal logic to prevent running more than once
}
}
LOGGER.info("[" + this.getClass().getSimpleName() + "] saving complete, closing repo.");
this.repo.close();
}
catch (InterruptedException ignore) { }
finally
{
this.closeLock.unlock();
}
}
}
@@ -21,19 +21,9 @@ public interface IDataSource<TDhLevel extends IDhLevel> extends IBaseDTO<DhSecti
{
DhSectionPos getSectionPos();
//===============//
// file handling //
//===============//
/** @return true if the data was changed */
boolean update(FullDataSourceV2 chunkData, TDhLevel level);
// still used by RenderSource, remove once that's been changed
@Deprecated
void writeToStream(DhDataOutputStream outputStream, TDhLevel level) throws IOException;
//===========//
@@ -1,33 +1,13 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.fullDatafile;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo;
import com.seibel.distanthorizons.core.sql.repo.LegacyFullDataRepo;
import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV1Repo;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
@@ -35,34 +15,53 @@ import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.locks.ReentrantLock;
public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler<FullDataSourceV1, IDhLevel>
public class FullDataSourceProviderV1<TDhLevel extends IDhLevel>
implements AutoCloseable
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
protected final ReentrantLock closeLock = new ReentrantLock();
protected volatile boolean isShutdown = false;
protected final TDhLevel level;
protected final File saveDir;
public final FullDataSourceV1Repo repo;
//=============//
// constructor //
//=============//
public FullDataSourceProviderV1(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride)
public FullDataSourceProviderV1(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride)
{
super(level, saveStructure, saveDirOverride);
this.level = level;
this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride;
if (!this.saveDir.exists() && !this.saveDir.mkdirs())
{
LOGGER.warn("Unable to create full data folder, file saving may fail.");
}
this.repo = this.createRepo();
}
//====================//
// Abstract overrides //
//====================//
//==================//
// abstract methods //
//==================//
@Override
protected AbstractLegacyDataSourceRepo createRepo()
/** When this is called the parent folders should be created */
protected FullDataSourceV1Repo createRepo()
{
try
{
return new LegacyFullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
return new FullDataSourceV1Repo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
}
catch (SQLException e)
{
@@ -72,32 +71,61 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler<Fu
}
}
@Override
protected FullDataSourceV1 createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException
protected FullDataSourceV1 createDataSourceFromDto(FullDataSourceV1DTO dto) throws InterruptedException, IOException
{
FullDataSourceV1 dataSource = FullDataSourceV1.createEmpty(dto.pos);
dataSource.populateFromStream(dto, dto.getInputStream(), this.level);
return dataSource;
}
/** Creates a new data source using any DTOs already present in the database. */
@Deprecated
@Override
protected FullDataSourceV1 createNewDataSourceFromExistingDtos(DhSectionPos pos) { return null; }
@Deprecated
@Override
protected FullDataSourceV1 makeEmptyDataSource(DhSectionPos pos) { return null; }
//===================//
// extension methods //
//===================//
//==============//
// data reading //
//==============//
@Deprecated
@Override
public void writeDataSourceToFile(FullDataSourceV1 fullDataSource) throws IOException
{ throw new UnsupportedOperationException("Deprecated"); }
/**
* Returns the {@link FullDataSourceV1} for the given section position. <Br>
* The returned data source may be null if there was a problem. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
public CompletableFuture<FullDataSourceV1> getAsync(DhSectionPos pos)
{
ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
return CompletableFuture.supplyAsync(() -> this.get(pos), executor);
}
/**
* Should only be used in internal file handler methods where we are already running on a file handler thread.
* Can return null.
* @see FullDataSourceProviderV1#getAsync(DhSectionPos)
*/
@Nullable
public FullDataSourceV1 get(DhSectionPos pos)
{
FullDataSourceV1 dataSource = null;
try
{
FullDataSourceV1DTO dto = this.repo.getByKey(pos);
if (dto != null)
{
// load from file
dataSource = this.createDataSourceFromDto(dto);
}
}
catch (InterruptedException ignore) { }
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e);
}
return dataSource;
}
@@ -105,13 +133,13 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler<Fu
// migration //
//===========//
public int getDataSourceMigrationCount() { return ((LegacyFullDataRepo) this.repo).getMigrationCount(); }
public int getDataSourceMigrationCount() { return this.repo.getMigrationCount(); }
public ArrayList<FullDataSourceV1> getDataSourcesToMigrate(int limit)
{
ArrayList<FullDataSourceV1> dataSourceList = new ArrayList<>();
ArrayList<DhSectionPos> migrationPosList = ((LegacyFullDataRepo) this.repo).getPositionsToMigrate(limit);
ArrayList<DhSectionPos> migrationPosList = ((FullDataSourceV1Repo) this.repo).getPositionsToMigrate(limit);
for (int i = 0; i < migrationPosList.size(); i++)
{
DhSectionPos pos = migrationPosList.get(i);
@@ -125,7 +153,29 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler<Fu
return dataSourceList;
}
public void markMigrationFailed(DhSectionPos pos) { ((LegacyFullDataRepo) this.repo).markMigrationFailed(pos); }
public void markMigrationFailed(DhSectionPos pos) { ((FullDataSourceV1Repo) this.repo).markMigrationFailed(pos); }
//=========//
// cleanup //
//=========//
@Override
public void close()
{
try
{
this.closeLock.lock();
this.isShutdown = true;
LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "].");
this.repo.close();
}
finally
{
this.closeLock.unlock();
}
}
}
@@ -1,45 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.renderfile;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo;
import java.util.concurrent.CompletableFuture;
/**
* Handles reading, writing, and updating {@link ColumnRenderSource}'s. <br>
* Should be backed by a database handled by a {@link RenderDataRepo}.
*
* @deprecated an interface isn't necessary for the single render source provider we have
*/
@Deprecated
public interface IRenderSourceProvider extends AutoCloseable
{
CompletableFuture<ColumnRenderSource> getAsync(DhSectionPos pos);
void updateDataSource(FullDataSourceV2 dataSource);
/** Deletes any data stored in the render cache so it can be re-created */
void deleteRenderCache();
}
@@ -1,173 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.renderfile;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo;
import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO;
import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.*;
public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler<ColumnRenderSource, IDhClientLevel> implements IRenderSourceProvider
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private final F3Screen.NestedMessage threadPoolMsg;
public final FullDataSourceProviderV2 fullDataSourceProvider;
//=============//
// constructor //
//=============//
public RenderSourceFileHandler(FullDataSourceProviderV2 sourceProvider, IDhClientLevel clientLevel, AbstractSaveStructure saveStructure)
{
super(clientLevel, saveStructure);
this.fullDataSourceProvider = sourceProvider;
this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log);
}
//====================//
// Abstract overrides //
//====================//
@Override
protected AbstractLegacyDataSourceRepo createRepo()
{
try
{
return new RenderDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME);
}
catch (SQLException e)
{
// should only happen if there is an issue with the database (it's locked or can't be created if missing)
// or the database update failed
throw new RuntimeException(e);
}
}
@Override
protected ColumnRenderSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException
{ return ColumnRenderSourceLoader.INSTANCE.loadRenderSource(dto, dto.getInputStream(), this.level); }
@Override
protected ColumnRenderSource createNewDataSourceFromExistingDtos(DhSectionPos pos)
{
ColumnRenderSource renderDataSource;
try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(pos))
{
renderDataSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level);
}
catch (Exception e) { throw new RuntimeException(e); }
return renderDataSource;
}
@Override
protected ColumnRenderSource makeEmptyDataSource(DhSectionPos pos)
{ return ColumnRenderSource.createEmptyRenderSource(pos); }
//=====================//
// extension overrides //
//=====================//
@Override
public void updateDataSource(FullDataSourceV2 inputDataSource)
{
// TODO once the legacy data provider has been replaced this can be removed
this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource);
}
//=========//
// F3 menu //
//=========//
/** Returns what should be displayed in Minecraft's F3 debug menu */
private String[] f3Log()
{
ThreadPoolExecutor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor();
String fileQueueSize = (fileExecutor != null) ? fileExecutor.getQueue().size()+"" : "-";
String fileCompletedTaskSize = (fileExecutor != null) ? fileExecutor.getCompletedTaskCount()+"" : "-";
ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor();
String updateQueueSize = (updateExecutor != null) ? updateExecutor.getQueue().size()+"" : "-";
String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-";
int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount();
ArrayList<String> lines = new ArrayList<>();
lines.add("File Handler [" + this.level.getLevelWrapper().getDimensionType().getDimensionName() + "]");
lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")");
lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")");
lines.add(" Level Unsaved #: " + this.level.getUnsavedDataSourceCount());
if (unsavedDataSourceCount != -1)
{
lines.add(" File Handler Unsaved #: " + unsavedDataSourceCount);
}
lines.add(" Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size());
lines.add(" Unsaved render sources: " + this.unsavedDataSourceBySectionPos.size());
return lines.toArray(new String[0]);
}
//=====================//
// shutdown / clearing //
//=====================//
public void close()
{
super.close();
this.threadPoolMsg.close();
}
public void deleteRenderCache() { this.repo.deleteAll(); }
}
@@ -26,7 +26,6 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler;
import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
@@ -36,13 +35,16 @@ import com.seibel.distanthorizons.core.render.LodQuadTree;
import com.seibel.distanthorizons.core.render.RenderBufferHandler;
import com.seibel.distanthorizons.core.render.renderer.LodRenderer;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper;
import org.apache.logging.log4j.Logger;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicReference;
public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc<FullDataSourceV2>
@@ -50,7 +52,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
private final IDhClientLevel parentClientLevel;
private final IDhClientLevel clientLevel;
public final FullDataSourceProviderV2 fullDataSourceProvider;
public final AtomicReference<ClientRenderState> ClientRenderStateRef = new AtomicReference<>();
@@ -63,12 +65,12 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
// constructor //
//=============//
public ClientLevelModule(IDhClientLevel parentClientLevel)
public ClientLevelModule(IDhClientLevel clientLevel)
{
this.parentClientLevel = parentClientLevel;
this.clientLevel = clientLevel;
this.f3Message = new F3Screen.NestedMessage(this::f3Log);
this.fullDataSourceProvider = this.parentClientLevel.getFullDataProvider();
this.fullDataSourceProvider = this.clientLevel.getFullDataProvider();
this.fullDataSourceProvider.dateSourceUpdateListeners.add(this);
}
@@ -102,14 +104,14 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
return;
}
IClientLevelWrapper clientLevelWrapper = this.parentClientLevel.getClientLevelWrapper();
IClientLevelWrapper clientLevelWrapper = this.clientLevel.getClientLevelWrapper();
if (clientLevelWrapper == null)
{
return;
}
clientRenderState.close();
clientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure());
clientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure());
if (!this.ClientRenderStateRef.compareAndSet(null, clientRenderState))
{
//FIXME: How to handle this?
@@ -141,7 +143,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
/** @return if the {@link ClientRenderState} was successfully swapped */
public boolean startRenderer(IClientLevelWrapper clientLevelWrapper)
{
ClientRenderState ClientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure());
// TODO why are we passing in a level wrapper? Our client level is already defined.
ClientRenderState ClientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure());
if (!this.ClientRenderStateRef.compareAndSet(null, ClientRenderState))
{
LOGGER.warn("Failed to start renderer due to concurrency");
@@ -208,15 +211,14 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
// data handling //
//===============//
public CompletableFuture<Void> updateDataSourcesAsync(FullDataSourceV2 data) { return this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); }
public CompletableFuture<Void> updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientLevel.getFullDataProvider().updateDataSourceAsync(data); }
@Override
public void OnDataSourceUpdated(FullDataSourceV2 updatedFullDataSource)
{
// if rendering also update the render sources
// if rendering, also update the render sources
ClientRenderState ClientRenderState = this.ClientRenderStateRef.get();
if (ClientRenderState != null)
{
ClientRenderState.renderSourceFileHandler.updateDataSource(updatedFullDataSource);
ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos());
}
}
@@ -254,19 +256,40 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
// misc helper functions //
//=======================//
/** Returns what should be displayed in Minecraft's F3 debug menu */
protected String[] f3Log()
private String[] f3Log()
{
String dimName = this.parentClientLevel.getClientLevelWrapper().getDimensionType().getDimensionName();
ClientRenderState renderState = this.ClientRenderStateRef.get();
if (renderState == null)
String dimName = this.clientLevel.getLevelWrapper().getDimensionType().getDimensionName();
boolean rendererActive = this.ClientRenderStateRef.get() != null;
ThreadPoolExecutor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor();
String fileQueueSize = (fileExecutor != null) ? fileExecutor.getQueue().size()+"" : "-";
String fileCompletedTaskSize = (fileExecutor != null) ? fileExecutor.getCompletedTaskCount()+"" : "-";
ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor();
String updateQueueSize = (updateExecutor != null) ? updateExecutor.getQueue().size()+"" : "-";
String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-";
int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount();
ArrayList<String> lines = new ArrayList<>();
lines.add("");
lines.add("level [" + dimName + "] rendering: " + (rendererActive ? "Active" : "Inactive"));
// TODO a lot of these items only need to be rendered once, but we don't currently have a way of doing that, so only add them for the rendered level
if (rendererActive)
{
return new String[]{"level @ " + dimName + ": Inactive"};
}
else
{
return new String[]{"level @ " + dimName + ": Active"};
lines.add("File Handler [" + dimName + "]");
lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")");
lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")");
lines.add(" Level Unsaved #: " + this.clientLevel.getUnsavedDataSourceCount());
if (unsavedDataSourceCount != -1)
{
lines.add(" File Handler Unsaved #: " + unsavedDataSourceCount);
}
lines.add(" Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size());
}
return lines.toArray(new String[0]);
}
public void clearRenderCache()
@@ -299,7 +322,6 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
public final IClientLevelWrapper clientLevelWrapper;
public final LodQuadTree quadtree;
public final RenderSourceFileHandler renderSourceFileHandler;
public final LodRenderer renderer;
public ClientRenderState(
@@ -307,12 +329,11 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
AbstractSaveStructure saveStructure)
{
this.clientLevelWrapper = clientLevelWrapper;
this.renderSourceFileHandler = new RenderSourceFileHandler(fullDataSourceProvider, dhClientLevel, saveStructure);
this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get() * LodUtil.CHUNK_WIDTH * 2,
// initial position is (0,0) just in case the player hasn't loaded in yet, the tree will be moved once the level starts ticking
0, 0,
this.renderSourceFileHandler.fullDataSourceProvider, this.renderSourceFileHandler);
fullDataSourceProvider);
RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree);
this.renderer = new LodRenderer(renderBufferHandler);
@@ -326,7 +347,6 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle
this.renderer.close();
this.quadtree.close();
this.renderSourceFileHandler.close();
}
}
@@ -24,7 +24,6 @@ import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
@@ -60,7 +59,6 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
public final int blockRenderDistanceDiameter;
private final FullDataSourceProviderV2 fullDataSourceProvider;
private final IRenderSourceProvider renderSourceProvider;
/**
* This holds every {@link DhSectionPos} that should be reloaded next tick. <br>
@@ -91,14 +89,12 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
public LodQuadTree(
IDhClientLevel level, int viewDiameterInBlocks,
int initialPlayerBlockX, int initialPlayerBlockZ,
FullDataSourceProviderV2 fullDataSourceProvider,
IRenderSourceProvider renderSourceProvider)
FullDataSourceProviderV2 fullDataSourceProvider)
{
super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL);
this.level = level;
this.fullDataSourceProvider = fullDataSourceProvider;
this.renderSourceProvider = renderSourceProvider;
this.blockRenderDistanceDiameter = viewDiameterInBlocks;
this.horizontalScaleChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, (newHorizontalScale) -> this.onHorizontalQualityChange());
@@ -164,7 +160,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
LodRenderSection renderSection = this.getValue(pos);
if (renderSection != null)
{
renderSection.reload(this.renderSourceProvider);
renderSection.reload(this.fullDataSourceProvider);
}
}
catch (IndexOutOfBoundsException e)
@@ -313,7 +309,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
{
// prepare this section for rendering
// TODO this should fire for the lowest detail level first to improve loading speed
renderSection.loadRenderSource(this.renderSourceProvider, this.level);
renderSection.loadRenderSource(this.fullDataSourceProvider, this.level);
// wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections
if (!parentRenderSectionIsEnabled && renderSection.canRenderNow())
@@ -435,10 +431,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
{
try
{
LOGGER.info("Clearing render cache...");
// delete the cache first so the nodes won't accidentally try re-loading the old data
this.renderSourceProvider.deleteRenderCache();
LOGGER.info("Disposing render data...");
// clear the tree
Iterator<QuadNode<LodRenderSection>> nodeIterator = this.nodeIterator();
@@ -452,7 +445,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements AutoClose
}
}
LOGGER.info("Render cache invalidated, please wait a moment for everything to reload...");
LOGGER.info("Render data cleared, please wait a moment for everything to reload...");
}
catch (Exception e)
{
@@ -20,11 +20,12 @@
package com.seibel.distanthorizons.core.render;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
@@ -57,13 +58,13 @@ public class LodRenderSection implements IDebugRenderable
private boolean isRenderingEnabled = false;
/**
* If this is true, then {@link LodRenderSection#reload(IRenderSourceProvider)} was called while
* a {@link IRenderSourceProvider} was already being loaded.
* If this is true, then {@ link LodRenderSection#reload(IRenderSourceProvider)} was called while
* a {@ link IRenderSourceProvider} was already being loaded.
*/
private boolean reloadRenderSourceOnceLoaded = false;
private IRenderSourceProvider renderSourceProvider = null;
private CompletableFuture<ColumnRenderSource> renderSourceLoadFuture;
private FullDataSourceProviderV2 fullDataSourceProvider = null;
private CompletableFuture<FullDataSourceV2> fullDataSourceLoadFuture;
private ColumnRenderSource renderSource;
private IDhClientLevel level = null;
@@ -120,17 +121,17 @@ public class LodRenderSection implements IDebugRenderable
//=============//
/** does nothing if a render source is already loaded or in the process of loading */
public void loadRenderSource(IRenderSourceProvider renderDataProvider, IDhClientLevel level)
public void loadRenderSource(FullDataSourceProviderV2 fullDataSourceProvider, IDhClientLevel level)
{
this.renderSourceProvider = renderDataProvider;
this.fullDataSourceProvider = fullDataSourceProvider;
this.level = level;
if (this.renderSourceProvider == null)
if (this.fullDataSourceProvider == null)
{
LOGGER.warn("LodRenderSection [" + this.pos + "] called loadRenderSource with a empty source provider");
return;
}
// don't re-load or double load the render source
if (this.renderSource != null || this.renderSourceLoadFuture != null)
if (this.renderSource != null || this.fullDataSourceLoadFuture != null)
{
// since the render source has been loaded, make sure the render buffers are populated
// FIXME this is a duck tape solution, since the renderBufferRef should be populated elsewhere, but this does fix empty LODs when moving around the world
@@ -145,7 +146,7 @@ public class LodRenderSection implements IDebugRenderable
this.startLoadRenderSourceAsync();
}
public void reload(IRenderSourceProvider renderDataProvider)
public void reload(FullDataSourceProviderV2 fullDataSourceProvider)
{
// debug rendering
boolean showRenderSectionStatus = Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus.get();
@@ -160,8 +161,8 @@ public class LodRenderSection implements IDebugRenderable
}
this.renderSourceProvider = renderDataProvider;
if (this.renderSourceProvider == null)
this.fullDataSourceProvider = fullDataSourceProvider;
if (this.fullDataSourceProvider == null)
{
LOGGER.warn("LodRenderSection [" + this.pos + "] called reload with a empty source provider");
return;
@@ -173,7 +174,7 @@ public class LodRenderSection implements IDebugRenderable
return;
}
// wait for the current load future to finish before re-loading
if (this.renderSourceLoadFuture != null)
if (this.fullDataSourceLoadFuture != null)
{
this.reloadRenderSourceOnceLoaded = true;
return;
@@ -184,19 +185,22 @@ public class LodRenderSection implements IDebugRenderable
private void startLoadRenderSourceAsync()
{
this.renderSourceLoadFuture = this.renderSourceProvider.getAsync(this.pos);
this.renderSourceLoadFuture.whenComplete((renderSource, ex) ->
this.fullDataSourceLoadFuture = this.fullDataSourceProvider.getAsync(this.pos);
this.fullDataSourceLoadFuture.whenComplete((fullDataSource, ex) ->
{
this.renderSource = renderSource;
// this runs on the a file handler thread, so transforming the data
// here shouldn't cause any stutters
// (Although it might be good to have it on a separate thread anyway)
this.renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level);
this.lastNs = -1;
this.markBufferDirty();
if (this.reloadRenderSourceOnceLoaded)
{
this.reloadRenderSourceOnceLoaded = false;
this.reload(this.renderSourceProvider);
this.reload(this.fullDataSourceProvider);
}
this.renderSourceLoadFuture = null;
this.fullDataSourceLoadFuture = null;
});
}
@@ -213,7 +217,7 @@ public class LodRenderSection implements IDebugRenderable
public boolean canRenderNow()
{
if (this.renderSourceLoadFuture != null || this.buildRenderBufferFuture != null)
if (this.fullDataSourceLoadFuture != null || this.buildRenderBufferFuture != null)
{
// wait for loading to finish
return false;
@@ -278,7 +282,7 @@ public class LodRenderSection implements IDebugRenderable
}
/** @return true if this section is loaded and set to render */
public boolean canBuildBuffer() { return this.renderSourceLoadFuture == null && this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); }
public boolean canBuildBuffer() { return this.fullDataSourceLoadFuture == null && this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); }
private boolean isBufferOutdated() { return this.neighborUpdated || this.renderSource.localVersion.get() != this.lastSwapLocalVersion; }
/** @return true if this section is loaded and set to render */
@@ -296,10 +300,10 @@ public class LodRenderSection implements IDebugRenderable
this.disposeActiveBuffer = true;
this.renderSource = null;
if (this.renderSourceLoadFuture != null)
if (this.fullDataSourceLoadFuture != null)
{
this.renderSourceLoadFuture.cancel(true);
this.renderSourceLoadFuture = null;
this.fullDataSourceLoadFuture.cancel(true);
this.fullDataSourceLoadFuture = null;
}
}
@@ -473,7 +477,7 @@ public class LodRenderSection implements IDebugRenderable
return "LodRenderSection{" +
"pos=" + this.pos +
", lodRenderSource=" + this.renderSource +
", loadFuture=" + this.renderSourceLoadFuture +
", loadFuture=" + this.fullDataSourceLoadFuture +
", isRenderEnabled=" + this.isRenderingEnabled +
'}';
}
@@ -496,11 +500,11 @@ public class LodRenderSection implements IDebugRenderable
public void debugRender(DebugRenderer debugRenderer)
{
Color color = Color.red;
if (this.renderSourceProvider == null)
if (this.fullDataSourceProvider == null)
{
color = Color.black;
}
else if (this.renderSourceLoadFuture != null)
else if (this.fullDataSourceLoadFuture != null)
{
color = Color.yellow;
}
@@ -32,20 +32,12 @@ import java.io.InputStream;
import java.util.concurrent.atomic.AtomicLong;
/**
* Handles storing {@link ColumnRenderSource}'s and {@link FullDataSourceV1}'s in the database.
*
* @deprecated {@link ColumnRenderSource} should store the actual GPU buffer data,
* at that point this DTO should be just used for {@link FullDataSourceV1}
* and could be renamed to FullDataSourceV1DTO
* Handles storing{@link FullDataSourceV1}'s in the database.
*/
@Deprecated
public class LegacyDataSourceDTO implements IBaseDTO<DhSectionPos>
public class FullDataSourceV1DTO implements IBaseDTO<DhSectionPos>
{
public DhSectionPos pos;
public int checksum;
/** @deprecated the database now has a last modified date time that should be used instead */
@Deprecated
public AtomicLong dataVersion = new AtomicLong(Long.MAX_VALUE);
public byte dataDetailLevel;
public EDhApiWorldGenerationStep worldGenStep;
@@ -62,7 +54,7 @@ public class LegacyDataSourceDTO implements IBaseDTO<DhSectionPos>
// constructor //
//=============//
public LegacyDataSourceDTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray)
public FullDataSourceV1DTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray)
{
this.pos = pos;
this.checksum = checksum;
@@ -21,34 +21,55 @@ package com.seibel.distanthorizons.core.sql.repo;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo<DhSectionPos, LegacyDataSourceDTO>
public class FullDataSourceV1Repo extends AbstractDhRepo<DhSectionPos, FullDataSourceV1DTO>
{
public AbstractLegacyDataSourceRepo(String databaseType, String databaseLocation) throws SQLException
public static final String TABLE_NAME = "Legacy_FullData_V1";
//=============//
// constructor //
//=============//
public FullDataSourceV1Repo(String databaseType, String databaseLocation) throws SQLException
{
super(databaseType, databaseLocation, LegacyDataSourceDTO.class);
super(databaseType, databaseLocation, FullDataSourceV1DTO.class);
}
//===========//
// overrides //
//===========//
@Override
public String getTableName() { return TABLE_NAME; }
@Override
public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; }
//=======================//
// repo required methods //
//=======================//
@Override
public LegacyDataSourceDTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
public FullDataSourceV1DTO convertDictionaryToDto(Map<String, Object> objectMap) throws ClassCastException
{
String posString = (String) objectMap.get("DhSectionPos");
DhSectionPos pos = DhSectionPos.deserialize(posString);
// meta data
int checksum = (Integer) objectMap.get("Checksum");
long dataVersion = (Long) objectMap.get("DataVersion");
byte dataDetailLevel = (Byte) objectMap.get("DataDetailLevel");
String worldGenStepString = (String) objectMap.get("WorldGenStep");
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromName(worldGenStepString);
@@ -59,7 +80,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo<DhSect
// binary data
byte[] dataByteArray = (byte[]) objectMap.get("Data");
LegacyDataSourceDTO dto = new LegacyDataSourceDTO(
FullDataSourceV1DTO dto = new FullDataSourceV1DTO(
pos,
checksum, dataDetailLevel, worldGenStep,
dataType, binaryDataFormatVersion,
@@ -68,7 +89,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo<DhSect
}
@Override
public PreparedStatement createInsertStatement(LegacyDataSourceDTO dto) throws SQLException
public PreparedStatement createInsertStatement(FullDataSourceV1DTO dto) throws SQLException
{
String sql =
"INSERT INTO "+this.getTableName() + "\n" +
@@ -87,7 +108,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo<DhSect
statement.setObject(i++, dto.pos.serialize());
statement.setObject(i++, dto.checksum);
statement.setObject(i++, dto.dataVersion);
statement.setObject(i++, 0 /*dto.dataVersion*/);
statement.setObject(i++, dto.dataDetailLevel);
statement.setObject(i++, dto.worldGenStep);
statement.setObject(i++, dto.dataType);
@@ -99,7 +120,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo<DhSect
}
@Override
public PreparedStatement createUpdateStatement(LegacyDataSourceDTO dto) throws SQLException
public PreparedStatement createUpdateStatement(FullDataSourceV1DTO dto) throws SQLException
{
String sql =
"UPDATE "+this.getTableName()+" \n" +
@@ -119,7 +140,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo<DhSect
int i = 1;
statement.setObject(i++, dto.checksum);
statement.setObject(i++, dto.dataVersion);
statement.setObject(i++, 0 /*dto.dataVersion*/);
statement.setObject(i++, dto.dataDetailLevel);
statement.setObject(i++, dto.worldGenStep);
statement.setObject(i++, dto.dataType);
@@ -159,4 +180,58 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo<DhSect
}
//===========//
// migration //
//===========//
/** Returns how many positions need to be migrated over to the new version */
public int getMigrationCount()
{
Map<String, Object> resultMap = this.queryDictionaryFirst(
"select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1");
if (resultMap == null)
{
return 0;
}
else
{
int count = (int) resultMap.get("itemCount");
return count;
}
}
/** Returns the new "returnCount" positions that need to be migrated */
public ArrayList<DhSectionPos> getPositionsToMigrate(int returnCount)
{
ArrayList<DhSectionPos> list = new ArrayList<>();
List<Map<String, Object>> resultMapList = this.queryDictionary(
"select DhSectionPos " +
"from "+this.getTableName()+" " +
"WHERE MigrationFailed <> 1 " +
"LIMIT "+returnCount+";");
for (Map<String, Object> resultMap : resultMapList)
{
// returned in the format [sectionDetailLevel,x,z] IE [6,0,0]
DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos"));
list.add(sectionPos);
}
return list;
}
public void markMigrationFailed(DhSectionPos pos)
{
String sql =
"UPDATE "+this.getTableName()+" \n" +
"SET MigrationFailed = 1 \n" +
"WHERE DhSectionPos = '"+pos.serialize()+"'";
this.queryDictionaryFirst(sql);
}
}
@@ -1,102 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.sql.repo;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo
{
public static final String TABLE_NAME = "Legacy_FullData_V1";
public LegacyFullDataRepo(String databaseType, String databaseLocation) throws SQLException
{
super(databaseType, databaseLocation);
}
@Override
public String getTableName() { return TABLE_NAME; }
@Override
public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; }
//===========//
// migration //
//===========//
/** Returns how many positions need to be migrated over to the new version */
public int getMigrationCount()
{
Map<String, Object> resultMap = this.queryDictionaryFirst(
"select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1");
if (resultMap == null)
{
return 0;
}
else
{
int count = (int) resultMap.get("itemCount");
return count;
}
}
/** Returns the new "returnCount" positions that need to be migrated */
public ArrayList<DhSectionPos> getPositionsToMigrate(int returnCount)
{
ArrayList<DhSectionPos> list = new ArrayList<>();
List<Map<String, Object>> resultMapList = this.queryDictionary(
"select DhSectionPos " +
"from "+this.getTableName()+" " +
"WHERE MigrationFailed <> 1 " +
"LIMIT "+returnCount+";");
for (Map<String, Object> resultMap : resultMapList)
{
// returned in the format [sectionDetailLevel,x,z] IE [6,0,0]
DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos"));
list.add(sectionPos);
}
return list;
}
public void markMigrationFailed(DhSectionPos pos)
{
String sql =
"UPDATE "+this.getTableName()+" \n" +
"SET MigrationFailed = 1 \n" +
"WHERE DhSectionPos = '"+pos.serialize()+"'";
this.queryDictionaryFirst(sql);
}
}
@@ -1,43 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020-2023 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.sql.repo;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import java.sql.SQLException;
public class RenderDataRepo extends AbstractLegacyDataSourceRepo
{
public static final String TABLE_NAME = "DhRenderData";
public RenderDataRepo(String databaseType, String databaseLocation) throws SQLException
{
super(databaseType, databaseLocation);
}
@Override
public String getTableName() { return TABLE_NAME; }
@Override
public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; }
}