wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix empty data sources when moving in multiplayer or with N-sized world gen

Browse Source 
Increases Protocol version 9 -> 10
This commit is contained in:
James Seibel
2025-02-05 17:30:59 -06:00
parent cedaaa8a2e
commit 10a3840373
14 changed files with 581 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/java/com/seibel/distanthorizons/api/enums/worldGeneration/EDhApiWorldGenerationStep.java
+10
View File
@@ -20,6 +20,8 @@
package com.seibel.distanthorizons.api.enums.worldGeneration;
/**
* DOWN_SAMPLED, <br>
*
* EMPTY, <br>
* STRUCTURE_START, <br>
* STRUCTURE_REFERENCE, <br>
@@ -37,6 +39,14 @@ package com.seibel.distanthorizons.api.enums.worldGeneration;
*/
public enum EDhApiWorldGenerationStep
{
/**
* Only used when using N-sized world generators or server-side retrieval.
* This denotes that the given datasource was created using lower quality LOD data from above it in the quad tree. <br>
*
* This isn't a valid option for queuing world generation.
*/
DOWN_SAMPLED(-1, "down_sampled"),
EMPTY(0, "empty"),
STRUCTURE_START(1, "structure_start"),
STRUCTURE_REFERENCE(2, "structure_reference"),
api/src/main/java/com/seibel/distanthorizons/coreapi/ModInfo.java
+1 -1
View File
@@ -31,7 +31,7 @@ public final class ModInfo
public static final String DEDICATED_SERVER_INITIAL_PATH = "dedicated_server_initial";
/** Incremented every time any packets are added, changed or removed, with a few exceptions. */
public static final int PROTOCOL_VERSION = 9;
public static final int PROTOCOL_VERSION = 10;
public static final String WRAPPER_PACKET_PATH = "message";
/** The internal mod name */
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+165 -8
View File
@@ -116,7 +116,12 @@ public class FullDataSourceV2
public final LongArrayList[] dataPoints;
public boolean isEmpty;
public boolean applyToParent = false;
/** Will be null if we don't want to update this value in the DB */
@Nullable
public Boolean applyToParent = null;
/** Will be null if we don't want to update this value in the DB */
@Nullable
public Boolean applyToChildren = null;
/** should only be used by methods exposed via the DH API */
private boolean runApiChunkValidation = false;
@@ -301,10 +306,47 @@ public class FullDataSourceV2
if (inputDetailLevel == thisDetailLevel)
{
dataChanged = this.updateFromSameDetailLevel(inputDataSource, remappedIds);
// same detail level, propagate parent/children update flags from input
if (this.applyToParent != null || inputDataSource.applyToParent != null)
{
this.applyToParent =
// copy over application flag if either are set to continue propagating
(BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent))
// don't propagate past the top of the tree
&& (DhSectionPos.getDetailLevel(this.pos) < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL);
}
// null check to prevent setting a flag we don't want to save in the DB
if (this.applyToChildren != null || inputDataSource.applyToChildren != null)
{
this.applyToChildren =
(BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren))
// don't propagate past the bottom of the tree
&& (DhSectionPos.getDetailLevel(this.pos) > AbstractDataSourceHandler.MIN_SECTION_DETAIL_LEVEL);
}
}
else if (inputDetailLevel + 1 == thisDetailLevel)
{
dataChanged = this.updateFromOneBelowDetailLevel(inputDataSource, remappedIds);
// propagating up, parent will need changes
this.applyToParent =
dataChanged
&& (BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent))
&& (DhSectionPos.getDetailLevel(this.pos) < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL);
}
else if (inputDetailLevel - 1 == thisDetailLevel)
{
dataChanged = this.downsampleFromOneAboveDetailLevel(inputDataSource, remappedIds);
// propagating down, children will need changes
this.applyToChildren =
dataChanged
&& (BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren))
&& (DhSectionPos.getDetailLevel(this.pos) > AbstractDataSourceHandler.MIN_SECTION_DETAIL_LEVEL);
}
else
{
@@ -312,12 +354,9 @@ public class FullDataSourceV2
// and would lead to edge cases that don't necessarily need to be supported
// (IE what do you do when the input is smaller than a single datapoint in the receiving data source?)
// instead it's better to just percolate the updates up
throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"].");
throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+(thisDetailLevel-1)+"], ["+thisDetailLevel+"], or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"].");
}
// determine if this data source should be applied to its parent
this.applyToParent = (dataChanged && DhSectionPos.getDetailLevel(this.pos) < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL);
if (dataChanged)
{
// update the hash code
@@ -326,6 +365,7 @@ public class FullDataSourceV2
return dataChanged;
}
public boolean updateFromSameDetailLevel(FullDataSourceV2 inputDataSource, int[] remappedIds)
{
// both data sources should have the same detail level
@@ -348,9 +388,31 @@ public class FullDataSourceV2
{
byte thisGenState = this.columnGenerationSteps.getByte(index);
byte inputGenState = inputDataSource.columnGenerationSteps.getByte(index);
if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value
&& thisGenState <= inputGenState)
// determine if this column should be updated
boolean genStateAllowsUpdating = false;
// if the input is downsampled, we only want to replace empty or downsampled values
if (inputGenState == EDhApiWorldGenerationStep.DOWN_SAMPLED.value
&&
(
thisGenState == EDhApiWorldGenerationStep.EMPTY.value
|| thisGenState == EDhApiWorldGenerationStep.DOWN_SAMPLED.value
))
{
genStateAllowsUpdating = true;
}
// if the input is any other non-empty value,
// replace anything that is less-complete
else if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value
&& thisGenState <= inputGenState)
{
// don't apply less-complete generation data
genStateAllowsUpdating = true;
}
if (genStateAllowsUpdating)
{
// check if the data changed
if (this.dataPoints[index] == null)
@@ -830,6 +892,101 @@ public class FullDataSourceV2
return value;
}
/**
* Only downsamples into a given column if this data source doesn't
* already contain data in that column.
* This is done to prevent accidentally downsampling onto already present higher-detail data.
*/
public boolean downsampleFromOneAboveDetailLevel(FullDataSourceV2 inputDataSource, int[] remappedIds)
{
if (DhSectionPos.getDetailLevel(inputDataSource.pos) - 1 != DhSectionPos.getDetailLevel(this.pos))
{
throw new IllegalArgumentException("Input data source must be exactly 1 detail level above this data source. Expected [" + (DhSectionPos.getDetailLevel(this.pos) - 1) + "], received [" + DhSectionPos.getDetailLevel(inputDataSource.pos) + "].");
}
// input is one detail level higher (lower detail)
// so 1x1 input data points will be converted into 2x2 recipient data point
// determine where in this data source should be read from
// since the input is one detail level above this will be one of input position's 4 children
int minParentXPos = DhSectionPos.getX(DhSectionPos.getChildByIndex(inputDataSource.pos, 0));
int inputOffsetX = (DhSectionPos.getX(this.pos) == minParentXPos) ? 0 : (WIDTH / 2);
int minParentZPos = DhSectionPos.getZ(DhSectionPos.getChildByIndex(inputDataSource.pos, 0));
int inputOffsetZ = (DhSectionPos.getZ(this.pos) == minParentZPos) ? 0 : (WIDTH / 2);
// merge the input's data points
// into this data source's
boolean dataChanged = false;
for (int x = 0; x < WIDTH; x++)
{
for (int z = 0; z < WIDTH; z++)
{
// recipient index is 1-to-1
int recipientIndex = relativePosToIndex(x, z);
int inputX = (x / 2) + inputOffsetX;
int inputZ = (z / 2) + inputOffsetZ;
int inputIndex = relativePosToIndex(inputX, inputZ);
// world gen //
// a separate generation step needs to be used so can replace
// this data with higher-quality data when it is available
byte inputGenStep = EDhApiWorldGenerationStep.DOWN_SAMPLED.value;
this.columnGenerationSteps.set(recipientIndex, inputGenStep);
// world compression //
byte worldCompressionMode = inputDataSource.columnWorldCompressionMode.getByte(recipientIndex);
this.columnWorldCompressionMode.set(recipientIndex, worldCompressionMode);
// data points //
// check if this column should be downsampled
boolean downSampleColumn;
if (this.dataPoints[recipientIndex] == null)
{
downSampleColumn = true;
}
else
{
downSampleColumn = true; // assume empty until we find non-empty data
for (long dataPoint : this.dataPoints[recipientIndex])
{
if (dataPoint != FullDataPointUtil.EMPTY_DATA_POINT)
{
downSampleColumn = false;
break;
}
}
}
if (downSampleColumn)
{
LongArrayList inputDataArray = inputDataSource.dataPoints[inputIndex];
this.dataPoints[recipientIndex] = inputDataArray;
this.remapDataColumn(recipientIndex, remappedIds);
if (RUN_DATA_ORDER_VALIDATION)
{
throwIfDataColumnInWrongOrder(inputDataSource.pos, this.dataPoints[recipientIndex]);
}
dataChanged = true;
}
this.isEmpty = false;
}
}
return dataChanged;
}
//================//
@@ -977,7 +1134,7 @@ public class FullDataSourceV2
LongArrayList packedDataPoints = LodDataBuilder.convertApiDataPointListToPackedLongArray(columnDataPoints, this, 0);
// TODO there should be an "unknown" compression and generation step, or be defined via the datapoints
this.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT, EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS);
this.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.SURFACE, EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS);
return columnDataPoints;
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java
+2
View File
@@ -73,6 +73,8 @@ public class LodDataBuilder
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos);
dataSource.isEmpty = false;
// chunk updates always propagate up
dataSource.applyToParent = true;
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java
+242 -104
View File
@@ -70,7 +70,7 @@ public class FullDataSourceProviderV2
protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50;
/** how many parent update tasks can be in the queue at once */
protected static final int MAX_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Common.MultiThreading.numberOfThreads.get();
protected static int getMaxUpdateTaskCount() { return NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD* Config.Common.MultiThreading.numberOfThreads.get(); }
/** indicates how long the update queue thread should wait between queuing ticks */
protected static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250;
@@ -103,7 +103,7 @@ public class FullDataSourceProviderV2
* Tracks which positions are currently being updated
* to prevent duplicate concurrent updates.
*/
public final Set<Long> parentUpdatingPosSet = ConcurrentHashMap.newKeySet();
public final Set<Long> updatingPosSet = ConcurrentHashMap.newKeySet();
// TODO only run thread if modifications happened recently
/**
@@ -225,107 +225,8 @@ public class FullDataSourceProviderV2
targetBlockPos = MC_CLIENT.getPlayerBlockPos();
}
// queue parent updates
if (executor.getQueueSize() < MAX_UPDATE_TASK_COUNT
&& this.parentUpdatingPosSet.size() < MAX_UPDATE_TASK_COUNT)
{
// get the positions that need to be applied to their parents
LongArrayList parentUpdatePosList = this.repo.getPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), MAX_UPDATE_TASK_COUNT);
// combine updates together based on their parent
HashMap<Long, HashSet<Long>> updatePosByParentPos = new HashMap<>();
for (Long pos : parentUpdatePosList)
{
updatePosByParentPos.compute(DhSectionPos.getParentPos(pos), (parentPos, updatePosSet) ->
{
if (updatePosSet == null)
{
updatePosSet = new HashSet<>();
}
updatePosSet.add(pos);
return updatePosSet;
});
}
// queue the updates
for (Long parentUpdatePos : updatePosByParentPos.keySet())
{
// stop if there are already a bunch of updates queued
if (this.parentUpdatingPosSet.size() > MAX_UPDATE_TASK_COUNT
|| !this.parentUpdatingPosSet.add(parentUpdatePos))
{
break;
}
try
{
executor.execute(() ->
{
ReentrantLock parentWriteLock = this.updateLockProvider.getLock(parentUpdatePos);
boolean parentLocked = false;
try
{
//LOGGER.info("updating parent: "+parentUpdatePos);
// Locking the parent before the children should prevent deadlocks.
// TryLock is used instead of lock so this thread can handle a different update.
if (parentWriteLock.tryLock())
{
parentLocked = true;
this.lockedPosSet.add(parentUpdatePos);
// apply each child pos to the parent
for (Long childPos : updatePosByParentPos.get(parentUpdatePos))
{
ReentrantLock childReadLock = this.updateLockProvider.getLock(childPos);
try
{
childReadLock.lock();
this.lockedPosSet.add(childPos);
try (FullDataSourceV2 dataSource = this.get(childPos))
{
// can return null when the file handler is being shut down
if (dataSource != null)
{
this.updateDataSourceAtPos(parentUpdatePos, dataSource, false);
this.repo.setApplyToParent(childPos, false);
}
}
}
catch (Exception e)
{
LOGGER.error("Unexpected in update for parent pos: [" + DhSectionPos.toString(parentUpdatePos) + "] Error: [" + e.getMessage() + "].", e);
}
finally
{
childReadLock.unlock();
this.lockedPosSet.remove(childPos);
}
}
}
}
finally
{
if (parentLocked)
{
parentWriteLock.unlock();
this.lockedPosSet.remove(parentUpdatePos);
}
this.parentUpdatingPosSet.remove(parentUpdatePos);
}
});
}
catch (RejectedExecutionException ignore)
{ /* the executor was shut down, it should be back up shortly and able to accept new jobs */ }
catch (Exception e)
{
this.parentUpdatingPosSet.remove(parentUpdatePos);
throw e;
}
}
}
this.runParentUpdates(executor, targetBlockPos);
this.runChildUpdates(executor, targetBlockPos);
}
catch (InterruptedException ignored)
@@ -340,6 +241,243 @@ public class FullDataSourceProviderV2
LOGGER.info("Update thread ["+Thread.currentThread().getName()+"] terminated.");
}
/** will always apply updates */
private void runParentUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos)
{
int maxUpdateTaskCount = getMaxUpdateTaskCount();
// queue parent updates
if (executor.getQueueSize() < maxUpdateTaskCount
&& this.updatingPosSet.size() < maxUpdateTaskCount)
{
// get the positions that need to be applied to their parents
LongArrayList parentUpdatePosList = this.repo.getPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount);
// combine updates together based on their parent
HashMap<Long, HashSet<Long>> updatePosByParentPos = new HashMap<>();
for (Long pos : parentUpdatePosList)
{
updatePosByParentPos.compute(DhSectionPos.getParentPos(pos), (parentPos, updatePosSet) ->
{
if (updatePosSet == null)
{
updatePosSet = new HashSet<>();
}
updatePosSet.add(pos);
return updatePosSet;
});
}
// queue the updates
for (Long parentUpdatePos : updatePosByParentPos.keySet())
{
// stop if there are already a bunch of updates queued
if (this.updatingPosSet.size() > maxUpdateTaskCount
|| executor.getQueueSize() > maxUpdateTaskCount
|| !this.updatingPosSet.add(parentUpdatePos))
{
break;
}
try
{
executor.execute(() ->
{
ReentrantLock parentWriteLock = this.updateLockProvider.getLock(parentUpdatePos);
boolean parentLocked = false;
try
{
//LOGGER.info("updating parent: "+parentUpdatePos);
// Locking the parent before the children should prevent deadlocks.
// TryLock is used instead of lock so this thread can handle a different update.
if (parentWriteLock.tryLock())
{
parentLocked = true;
this.lockedPosSet.add(parentUpdatePos);
try (FullDataSourceV2 parentDataSource = this.get(parentUpdatePos))
{
// will return null if the file handler is shutting down
if (parentDataSource != null)
{
// apply each child pos to the parent
for (Long childPos : updatePosByParentPos.get(parentUpdatePos))
{
ReentrantLock childReadLock = this.updateLockProvider.getLock(childPos);
try
{
childReadLock.lock();
this.lockedPosSet.add(childPos);
try (FullDataSourceV2 childDataSource = this.get(childPos))
{
// can return null when the file handler is being shut down
if (childDataSource != null)
{
parentDataSource.update(childDataSource);
}
}
}
catch (Exception e)
{
LOGGER.error("Unexpected in parent update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e);
}
finally
{
childReadLock.unlock();
this.lockedPosSet.remove(childPos);
}
}
if (DhSectionPos.getDetailLevel(parentUpdatePos) < TOP_SECTION_DETAIL_LEVEL)
{
parentDataSource.applyToParent = true;
}
this.updateDataSourceAtPos(parentUpdatePos, parentDataSource, false);
for (Long childPos : updatePosByParentPos.get(parentUpdatePos))
{
this.repo.setApplyToParent(childPos, false);
}
}
}
}
}
finally
{
if (parentLocked)
{
parentWriteLock.unlock();
this.lockedPosSet.remove(parentUpdatePos);
}
this.updatingPosSet.remove(parentUpdatePos);
}
});
}
catch (RejectedExecutionException ignore)
{ /* the executor was shut down, it should be back up shortly and able to accept new jobs */ }
catch (Exception e)
{
this.updatingPosSet.remove(parentUpdatePos);
throw e;
}
}
}
}
/** stops if it finds any LOD data */
private void runChildUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos)
{
int maxUpdateTaskCount = getMaxUpdateTaskCount();
// queue child updates
if (executor.getQueueSize() < maxUpdateTaskCount
&& this.updatingPosSet.size() < maxUpdateTaskCount)
{
// get the positions that need to be applied to their children
LongArrayList childUpdatePosList = this.repo.getChildPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount);
// queue the updates
for (long parentUpdatePos : childUpdatePosList)
{
// stop if there are already a bunch of updates queued
if (this.updatingPosSet.size() > maxUpdateTaskCount
|| executor.getQueueSize() > maxUpdateTaskCount
|| !this.updatingPosSet.add(parentUpdatePos))
{
break;
}
try
{
executor.execute(() ->
{
ReentrantLock parentReadLock = this.updateLockProvider.getLock(parentUpdatePos);
boolean parentLocked = false;
try
{
//LOGGER.info("updating parent: "+parentUpdatePos);
// Locking the parent before the children should prevent deadlocks.
// TryLock is used instead of lock so this thread can handle a different update.
if (parentReadLock.tryLock())
{
parentLocked = true;
this.lockedPosSet.add(parentUpdatePos);
try (FullDataSourceV2 parentDataSource = this.get(parentUpdatePos))
{
// will return null if the file handler is shutting down
if (parentDataSource != null)
{
// apply parent to each child
for (int i = 0; i < 4; i++)
{
long childPos = DhSectionPos.getChildByIndex(parentUpdatePos, i);
ReentrantLock childWriteLock = this.updateLockProvider.getLock(childPos);
try
{
childWriteLock.lock();
this.lockedPosSet.add(childPos);
try (FullDataSourceV2 childDataSource = this.get(childPos))
{
// will return null if the file handler is shutting down
if (childDataSource != null)
{
childDataSource.update(parentDataSource);
// don't propagate child updates past the bottom of the tree
if (DhSectionPos.getDetailLevel(childPos) != DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL)
{
childDataSource.applyToChildren = true;
}
this.updateDataSourceAtPos(childPos, childDataSource, false);
}
}
}
catch (Exception e)
{
LOGGER.error("Unexpected in child update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e);
}
finally
{
childWriteLock.unlock();
this.lockedPosSet.remove(childPos);
}
}
this.repo.setApplyToChild(parentUpdatePos, false);
}
}
}
}
finally
{
if (parentLocked)
{
parentReadLock.unlock();
this.lockedPosSet.remove(parentUpdatePos);
}
this.updatingPosSet.remove(parentUpdatePos);
}
});
}
catch (RejectedExecutionException ignore)
{ /* the executor was shut down, it should be back up shortly and able to accept new jobs */ }
catch (Exception e)
{
this.updatingPosSet.remove(parentUpdatePos);
throw e;
}
}
}
}
@@ -645,7 +783,7 @@ public class FullDataSourceProviderV2
this.queuedUpdateCountsByPos
.forEach((pos, updateCountRef) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f + (updateCountRef.get() * 16f), 0.20f, Color.WHITE)); });
this.parentUpdatingPosSet
this.updatingPosSet
.forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f, 0.20f, Color.MAGENTA)); });
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java
+14 -6
View File
@@ -211,7 +211,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
PriorityTaskPicker.Executor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor();
if (updateExecutor == null || updateExecutor.getQueueSize() >= MAX_UPDATE_TASK_COUNT / 2)
if (updateExecutor == null || updateExecutor.getQueueSize() >= getMaxUpdateTaskCount() / 2)
{
// don't queue additional world gen requests if the updater is behind
return false;
@@ -219,7 +219,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
PriorityTaskPicker.Executor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor();
if (fileExecutor == null || fileExecutor.getQueueSize() >= MAX_UPDATE_TASK_COUNT / 2)
if (fileExecutor == null || fileExecutor.getQueueSize() >= getMaxUpdateTaskCount() / 2)
{
// don't queue additional world gen requests if the file handler is overwhelmed,
// otherwise LODs may not load in properly
@@ -313,7 +313,12 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
public boolean isFullyGenerated(ByteArrayList columnGenerationSteps)
{
return IntStream.range(0, columnGenerationSteps.size())
.noneMatch(i -> columnGenerationSteps.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value);
.noneMatch(i ->
{
byte value = columnGenerationSteps.getByte(i);
return value == EDhApiWorldGenerationStep.EMPTY.value
|| value == EDhApiWorldGenerationStep.DOWN_SAMPLED.value;
});
}
public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Generated Provider");
@@ -343,7 +348,8 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
// check if any positions are ungenerated
for (int i = 0; i < columnGenStepArray.size(); i++)
{
if (columnGenStepArray.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value)
if (columnGenStepArray.getByte(i) == EDhApiWorldGenerationStep.EMPTY.value
|| columnGenStepArray.getByte(i) == EDhApiWorldGenerationStep.DOWN_SAMPLED.value)
{
positionFullyGenerated = false;
break;
@@ -408,7 +414,8 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
}
}
if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY)
if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY
|| currentMinWorldGenStep == EDhApiWorldGenerationStep.DOWN_SAMPLED)
{
// queue the task
break checkWorldGenLoop;
@@ -417,7 +424,8 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
}
}
if (currentMinWorldGenStep != EDhApiWorldGenerationStep.EMPTY)
if (currentMinWorldGenStep != EDhApiWorldGenerationStep.EMPTY
&& currentMinWorldGenStep != EDhApiWorldGenerationStep.DOWN_SAMPLED)
{
// no world gen needed for this position
return;
core/src/main/java/com/seibel/distanthorizons/core/generation/BatchGenerator.java
+1 -1
View File
@@ -110,7 +110,7 @@ public class BatchGenerator implements IDhApiWorldGenerator
targetStep = EDhApiWorldGenerationStep.FEATURES;
break;
case INTERNAL_SERVER:
targetStep = EDhApiWorldGenerationStep.LIGHT; // TODO using something other than LIGHT would be good for clarity
targetStep = EDhApiWorldGenerationStep.LIGHT;
break;
}
core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java
+6
View File
@@ -26,6 +26,7 @@ import com.seibel.distanthorizons.api.objects.data.DhApiChunk;
import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker;
import com.seibel.distanthorizons.core.generation.tasks.InProgressWorldGenTaskGroup;
import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
@@ -463,6 +464,11 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
// set here so the API user doesn't have to pass in this value anywhere themselves
pooledDataSource.setRunApiChunkValidation(this.generator.runApiValidation());
// only apply to children if we aren't at the bottom of the tree
pooledDataSource.applyToChildren = DhSectionPos.getDetailLevel(pooledDataSource.getPos()) > DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL;
pooledDataSource.applyToParent = DhSectionPos.getDetailLevel(pooledDataSource.getPos()) < DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL + 12;
return this.generator.generateLod(
chunkPosMin.getX(), chunkPosMin.getZ(),
DhSectionPos.getX(requestPos), DhSectionPos.getZ(requestPos),
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java
+5
View File
@@ -240,6 +240,11 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
{
FullDataSourceV2DTO dataSourceDto = this.networkState.fullDataPayloadReceiver.decodeDataSourceAndReleaseBuffer(response.payload);
// set application flags based on the received detail level,
// this is needed so the data sources propagate correctly
dataSourceDto.applyToChildren = DhSectionPos.getDetailLevel(dataSourceDto.pos) > DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL;
dataSourceDto.applyToParent = DhSectionPos.getDetailLevel(dataSourceDto.pos) < DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL + 12;
AbstractExecutorService executor = ThreadPoolUtil.getNetworkCompressionExecutor();
if (executor == null)
{
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+22 -2
View File
@@ -29,6 +29,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListParent;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.network.INetworkObject;
import com.seibel.distanthorizons.core.util.BoolUtil;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.ListUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
@@ -40,6 +41,7 @@ import io.netty.buffer.ByteBuf;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.*;
@@ -70,7 +72,12 @@ public class FullDataSourceV2DTO
public byte dataFormatVersion;
public byte compressionModeValue;
public boolean applyToParent;
/** Will be null if we don't want to update this value in the DB */
@Nullable
public Boolean applyToParent;
/** Will be null if we don't want to update this value in the DB */
@Nullable
public Boolean applyToChildren;
public long lastModifiedUnixDateTime;
public long createdUnixDateTime;
@@ -105,6 +112,7 @@ public class FullDataSourceV2DTO
dto.lastModifiedUnixDateTime = dataSource.lastModifiedUnixDateTime;
dto.createdUnixDateTime = dataSource.createdUnixDateTime;
dto.applyToParent = dataSource.applyToParent;
dto.applyToChildren = dataSource.applyToChildren;
dto.levelMinY = dataSource.levelMinY;
}
@@ -195,6 +203,15 @@ public class FullDataSourceV2DTO
dataSource.isEmpty = false;
if (this.applyToParent != null)
{
dataSource.applyToParent = this.applyToParent;
}
if (this.applyToChildren != null)
{
dataSource.applyToChildren = this.applyToChildren;
}
return dataSource;
}
@@ -379,7 +396,8 @@ public class FullDataSourceV2DTO
out.writeByte(this.dataFormatVersion);
out.writeByte(this.compressionModeValue);
out.writeBoolean(this.applyToParent);
out.writeBoolean(BoolUtil.falseIfNull(this.applyToParent));
out.writeBoolean(BoolUtil.falseIfNull(this.applyToChildren));
out.writeLong(this.lastModifiedUnixDateTime);
out.writeLong(this.createdUnixDateTime);
@@ -406,6 +424,7 @@ public class FullDataSourceV2DTO
this.compressionModeValue = in.readByte();
this.applyToParent = in.readBoolean();
this.applyToChildren = in.readBoolean();
this.lastModifiedUnixDateTime = in.readLong();
this.createdUnixDateTime = in.readLong();
@@ -444,6 +463,7 @@ public class FullDataSourceV2DTO
.add("dataFormatVersion", this.dataFormatVersion)
.add("compressionModeValue", this.compressionModeValue)
.add("applyToParent", this.applyToParent)
.add("applyToChildren", this.applyToChildren)
.add("lastModifiedUnixDateTime", this.lastModifiedUnixDateTime)
.add("createdUnixDateTime", this.createdUnixDateTime)
.toString();
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
+83 -29
View File
@@ -25,6 +25,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.DbConnectionClosedException;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.util.BoolUtil;
import com.seibel.distanthorizons.core.util.ListUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
@@ -100,7 +101,9 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
byte dataFormatVersion = resultSet.getByte("DataFormatVersion");
byte compressionModeValue = resultSet.getByte("CompressionMode");
// while these values can be null in the DB, null would just equate to false
boolean applyToParent = (resultSet.getInt("ApplyToParent")) == 1;
boolean applyToChildren = (resultSet.getInt("ApplyToChildren")) == 1;
long lastModifiedUnixDateTime = resultSet.getLong("LastModifiedUnixDateTime");
long createdUnixDateTime = resultSet.getLong("CreatedUnixDateTime");
@@ -128,6 +131,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
dto.lastModifiedUnixDateTime = lastModifiedUnixDateTime;
dto.createdUnixDateTime = createdUnixDateTime;
dto.applyToParent = applyToParent;
dto.applyToChildren = applyToChildren;
dto.levelMinY = minY;
}
return dto;
@@ -138,13 +142,13 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
" DetailLevel, PosX, PosZ, \n" +
" MinY, DataChecksum, \n" +
" Data, ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" +
" DataFormatVersion, CompressionMode, ApplyToParent, \n" +
" DataFormatVersion, CompressionMode, ApplyToParent, ApplyToChildren, \n" +
" LastModifiedUnixDateTime, CreatedUnixDateTime) \n" +
"VALUES( \n" +
" ?, ?, ?, \n" +
" ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ? \n" +
");";
@Override
@@ -172,7 +176,9 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
statement.setByte(i++, dto.dataFormatVersion);
statement.setByte(i++, dto.compressionModeValue);
statement.setObject(i++, dto.applyToParent);
// if nothing is present assume we don't need/want to propagate updates
statement.setBoolean(i++, BoolUtil.falseIfNull(dto.applyToParent));
statement.setBoolean(i++, BoolUtil.falseIfNull(dto.applyToChildren));
statement.setLong(i++, System.currentTimeMillis()); // last modified unix time
statement.setLong(i++, System.currentTimeMillis()); // created unix time
@@ -180,29 +186,39 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
return statement;
}
private final String updateSqlTemplate =
"UPDATE "+this.getTableName()+" \n" +
"SET \n" +
" MinY = ? \n" +
" ,DataChecksum = ? \n" +
" ,Data = ? \n" +
" ,ColumnGenerationStep = ? \n" +
" ,ColumnWorldCompressionMode = ? \n" +
" ,Mapping = ? \n" +
" ,DataFormatVersion = ? \n" +
" ,CompressionMode = ? \n" +
" ,ApplyToParent = ? \n" +
" ,LastModifiedUnixDateTime = ? \n" +
" ,CreatedUnixDateTime = ? \n" +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
@Override
public PreparedStatement createUpdateStatement(FullDataSourceV2DTO dto) throws SQLException
{
PreparedStatement statement = this.createPreparedStatement(this.updateSqlTemplate);
// Dynamic string so we can update one, both, or neither
// of the applyTo... flags.
// This is necessary to prevent concurrent modifications when
// update propagation is run.
String updateSqlTemplate = (
"UPDATE "+this.getTableName()+" \n" +
"SET \n" +
" MinY = ? \n" +
" ,DataChecksum = ? \n" +
" ,Data = ? \n" +
" ,ColumnGenerationStep = ? \n" +
" ,ColumnWorldCompressionMode = ? \n" +
" ,Mapping = ? \n" +
" ,DataFormatVersion = ? \n" +
" ,CompressionMode = ? \n" +
// only update these values if they're present
(dto.applyToParent != null ? " ,ApplyToParent = ? \n" : "" ) +
(dto.applyToChildren != null ? " ,ApplyToChildren = ? \n" : "" ) +
" ,LastModifiedUnixDateTime = ? \n" +
" ,CreatedUnixDateTime = ? \n" +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?"
// intern should help reduce memory overhead due to this string being dynamic
).intern();
PreparedStatement statement = this.createPreparedStatement(updateSqlTemplate);
if (statement == null)
{
return null;
@@ -220,7 +236,14 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
statement.setByte(i++, dto.dataFormatVersion);
statement.setByte(i++, dto.compressionModeValue);
statement.setObject(i++, dto.applyToParent);
if (dto.applyToParent != null)
{
statement.setBoolean(i++, dto.applyToParent);
}
if (dto.applyToChildren != null)
{
statement.setBoolean(i++, dto.applyToChildren);
}
statement.setLong(i++, System.currentTimeMillis()); // last modified unix time
statement.setLong(i++, dto.createdUnixDateTime);
@@ -238,13 +261,26 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
// updates //
//=========//
/** should be be very similar to {@link FullDataSourceV2Repo#setApplyToChildrenSql} */
private final String setApplyToParentSql =
"UPDATE "+this.getTableName()+" \n" +
"SET ApplyToParent = ? \n" +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
public void setApplyToParent(long pos, boolean applyToParent)
{ this.setApplyToFlag(pos, applyToParent, true); }
/** should be be very similar to {@link FullDataSourceV2Repo#setApplyToParentSql} */
private final String setApplyToChildrenSql =
"UPDATE "+this.getTableName()+" \n" +
"SET ApplyToChildren = ? \n" +
"WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?";
public void setApplyToChild(long pos, boolean applyToChild)
{ this.setApplyToFlag(pos, applyToChild, false); }
private void setApplyToFlag(long pos, boolean applyFlag, boolean applyToParent)
{
PreparedStatement statement = this.createPreparedStatement(this.setApplyToParentSql);
String sql = applyToParent ? this.setApplyToParentSql : this.setApplyToChildrenSql;
PreparedStatement statement = this.createPreparedStatement(sql);
if (statement == null)
{
return;
@@ -254,7 +290,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
try
{
int i = 1;
statement.setBoolean(i++, applyToParent);
statement.setBoolean(i++, applyFlag);
int detailLevel = DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
statement.setInt(i++, detailLevel);
@@ -272,7 +308,10 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
}
}
private final String getPositionsToUpdateSql =
/** should be be very similar to {@link FullDataSourceV2Repo#getChildPositionsToUpdateSql} */
private final String getParentPositionsToUpdateSql =
"SELECT DetailLevel, PosX, PosZ, " +
" (sqrt(pow(PosX - ?, 2) + pow(PosZ - ?, 2))) AS Distance " +
"FROM "+this.getTableName()+" " +
@@ -280,11 +319,25 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
"ORDER BY Distance ASC " +
"LIMIT ?; ";
public LongArrayList getPositionsToUpdate(int targetBlockPosX, int targetBlockPosZ, int returnCount)
{ return this.getPositionsToUpdate(targetBlockPosX, targetBlockPosZ, returnCount, true); }
/** should be be very similar to {@link FullDataSourceV2Repo#getParentPositionsToUpdateSql} */
private final String getChildPositionsToUpdateSql =
"SELECT DetailLevel, PosX, PosZ, " +
" (sqrt(pow(PosX - ?, 2) + pow(PosZ - ?, 2))) AS Distance " +
"FROM "+this.getTableName()+" " +
"WHERE ApplyToChildren = 1 " +
"ORDER BY Distance ASC " +
"LIMIT ?; ";
public LongArrayList getChildPositionsToUpdate(int targetBlockPosX, int targetBlockPosZ, int returnCount)
{ return this.getPositionsToUpdate(targetBlockPosX, targetBlockPosZ, returnCount, false); }
private LongArrayList getPositionsToUpdate(int targetBlockPosX, int targetBlockPosZ, int returnCount, boolean getParentUpdates)
{
LongArrayList list = new LongArrayList();
PreparedStatement statement = this.createPreparedStatement(this.getPositionsToUpdateSql);
String sql = getParentUpdates ? this.getParentPositionsToUpdateSql : this.getChildPositionsToUpdateSql;
PreparedStatement statement = this.createPreparedStatement(sql);
if (statement == null)
{
return list;
@@ -321,6 +374,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
}
private final String getColumnGenerationStepSql =
"select ColumnGenerationStep, CompressionMode " +
"from "+this.getTableName()+" " +
core/src/main/java/com/seibel/distanthorizons/core/util/BoolUtil.java
+20
View File
@@ -0,0 +1,20 @@
package com.seibel.distanthorizons.core.util;
public class BoolUtil
{
/** Used to prevent null {@link Boolean} objects in if statements */
public static boolean falseIfNull(Boolean value)
{
if (value == null)
{
// default to false since null doesn't mean true in any context
// (Even in JavaScript)
return false;
}
else
{
return value;
}
}
}
core/src/main/resources/sqlScripts/0080-sqlite-addApplyToChildrenColumn.sql
+9
View File
@@ -0,0 +1,9 @@
-- Applying to children is needed to fix a bug with N-sized generation.
-- If we don't fill the whole tree with data, it's possible to render empty/incomplete LODs, which looks bad.
alter table FullData add column ApplyToChildren BIT NULL;
--batch--
-- significantly speeds up update handling
create index FullDataApplyToChildrenIndex on FullData (ApplyToChildren) where ApplyToChildren = 1;
core/src/main/resources/sqlScripts/scriptList.txt
+1
View File
@@ -7,3 +7,4 @@
0050-sqlite-addApplyToParentIndex.sql
0060-sqlite-createChunkHashTable.sql
0070-sqlite-createBeaconBeamTable.sql
0080-sqlite-addApplyToChildrenColumn.sql