wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'adjData'

Browse Source
This commit is contained in:
James Seibel
2025-11-14 07:46:37 -06:00
parent 6bfcf36687 b82a59ecbc
commit eea5198fb6
68 changed files with 3677 additions and 2935 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java
+4
View File
@@ -44,6 +44,10 @@ public enum EDhApiGpuUploadMethod
/** Fast rendering but may stutter when uploading. */
SUB_DATA(false, false),
/** Don't upload, only should be used for debugging */
@Deprecated // TODO remove before release
NONE(false, false),
/**
* May end up storing buffers in System memory. <br>
* Fast rending if in GPU memory, slow if in system memory, <br>
api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java
+1 -4
View File
@@ -64,10 +64,7 @@ public enum EDhApiMaxHorizontalResolution
/** How wide each LOD DataPoint is */
public final int dataPointWidth;
/**
* This is the same as detailLevel in LodQuadTreeNode,
* lowest is 0 highest is 9
*/
/** This is the same as detailLevel in LodQuadTreeNode */
public final byte detailLevel;
/* Start/End X/Z give the block positions
core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java
Vendored
+1 -2
View File
@@ -49,7 +49,6 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import com.seibel.distanthorizons.core.util.math.Vec3d;
import com.seibel.distanthorizons.core.util.math.Vec3i;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.LogManager;
import com.seibel.distanthorizons.core.logging.DhLogger;
import org.jetbrains.annotations.Nullable;
@@ -259,7 +258,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo
//===============================//
FullDataPointIdMap mapping = dataSource.mapping;
LongArrayList dataColumn = dataSource.get(relativePos.x, relativePos.z);
LongArrayList dataColumn = dataSource.getColumnAtRelPos(relativePos.x, relativePos.z);
if (dataColumn != null)
{
int dataColumnIndexCount = dataColumn.size();
core/src/main/java/com/seibel/distanthorizons/core/config/Config.java
+7
View File
@@ -1083,6 +1083,13 @@ public class Config
+ "")
.build();
public static ConfigEntry<EDhApiGpuUploadMethod> glUploadMode = new ConfigEntry.Builder<EDhApiGpuUploadMethod>()
.set(EDhApiGpuUploadMethod.AUTO)
.comment(""
+ "\n"
+ "")
.build();
}
public static class ColumnBuilderDebugging
core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java
+1
View File
@@ -123,6 +123,7 @@ public class ConfigHandler
this.initNestedClass(Config.class, ""); // Init root category
this.configFileHandler.loadFromFile();
this.runMinMaxValidation = !Config.Client.Advanced.Debugging.allowUnsafeValues.get();
this.isLoaded = true;
LOGGER.info("[" + ModInfo.NAME + "] Config initialised");
core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandler.java
+3 -3
View File
@@ -240,7 +240,7 @@ public class ConfigFileHandler
else if (entry.getTrueValue() == null)
{
// TODO when can this happen?
throw new IllegalArgumentException("Entry [" + entry.getNameAndCategory() + "] is null, this may be a problem with [" + ModInfo.NAME + "]. Please contact the authors.");
throw new IllegalArgumentException("BlockBiomeWrapperPair [" + entry.getNameAndCategory() + "] is null, this may be a problem with [" + ModInfo.NAME + "]. Please contact the authors.");
}
workConfig.set(entry.getNameAndCategory(), ConfigTypeConverters.attemptToConvertToString(entry.getType(), entry.getTrueValue()));
@@ -287,13 +287,13 @@ public class ConfigFileHandler
if (entry.getTrueValue() == null)
{
LOGGER.warn("Entry [" + entry.getNameAndCategory() + "] returned as null from the config. Using default value.");
LOGGER.warn("BlockBiomeWrapperPair [" + entry.getNameAndCategory() + "] returned as null from the config. Using default value.");
entry.setWithoutFiringEvents(entry.getDefaultValue());
}
}
catch (Exception e)
{
LOGGER.warn("Entry [" + entry.getNameAndCategory() + "] had an invalid value when loading the config. Using default value.");
LOGGER.warn("BlockBiomeWrapperPair [" + entry.getNameAndCategory() + "] had an invalid value when loading the config. Using default value.");
entry.setWithoutFiringEvents(entry.getDefaultValue());
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java
+151
View File
@@ -0,0 +1,151 @@
package com.seibel.distanthorizons.core.dataObjects;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import java.util.concurrent.ConcurrentHashMap;
/**
* A pooled compound key between the biome and blockState. <br>
* These objects are pooled since we will need this compound key
* many times.
*
* @see FullDataPointIdMap
* @see IBlockStateWrapper
* @see IBiomeWrapper
*/
public class BlockBiomeWrapperPair
{
private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class);
/** two levels are present so we don't need to use a key object */
private static final ConcurrentHashMap<IBlockStateWrapper, ConcurrentHashMap<IBiomeWrapper, BlockBiomeWrapperPair>> CACHED_PAIR_BY_BIOME_BY_BLOCK = new ConcurrentHashMap<>();
public final IBiomeWrapper biome;
public final IBlockStateWrapper blockState;
private int hashCode = 0;
private boolean hashGenerated = false;
private String serialString = null;
//=============//
// constructor //
//=============//
public static BlockBiomeWrapperPair get(IBlockStateWrapper blockState, IBiomeWrapper biome)
{
// check for existing entry
ConcurrentHashMap<IBiomeWrapper, BlockBiomeWrapperPair> pairByBiomeWrapper = CACHED_PAIR_BY_BIOME_BY_BLOCK.get(blockState);
if (pairByBiomeWrapper != null)
{
BlockBiomeWrapperPair pair = pairByBiomeWrapper.get(biome);
if (pair != null)
{
return pair;
}
}
// Lazily create the inner map and new BlockBiomeWrapperPair
return CACHED_PAIR_BY_BIOME_BY_BLOCK
.computeIfAbsent(blockState, newBlockState -> new ConcurrentHashMap<>())
.computeIfAbsent(biome, newBiome -> new BlockBiomeWrapperPair(biome, blockState));
}
private BlockBiomeWrapperPair(IBiomeWrapper biome, IBlockStateWrapper blockState)
{
this.biome = biome;
this.blockState = blockState;
}
//===========//
// overrides //
//===========//
/**
* Reminder: this hash code won't always be unique, collisions can occur;
* because of that this hash shouldn't be the only unique identifier for this object.
*/
@Override
public int hashCode()
{
// cache the hash code to improve speed
if (!this.hashGenerated)
{
this.hashCode = generateHashCode(this);
this.hashGenerated = true;
}
return this.hashCode;
}
private static int generateHashCode(BlockBiomeWrapperPair pair) { return generateHashCode(pair.biome, pair.blockState); }
private static int generateHashCode(IBiomeWrapper biome, IBlockStateWrapper blockState)
{
final int prime = 31;
int result = 1;
// the biome and blockstate hashcode should be already calculated by the time
// we get here, so this operation should be very fast
result = prime * result + (biome == null ? 0 : biome.hashCode());
result = prime * result + (blockState == null ? 0 : blockState.hashCode());
return result;
}
@Override
public boolean equals(Object otherObj)
{
if (otherObj == this)
{
return true;
}
if (!(otherObj instanceof BlockBiomeWrapperPair))
{
return false;
}
BlockBiomeWrapperPair other = (BlockBiomeWrapperPair) otherObj;
return other.biome.getSerialString().equals(this.biome.getSerialString())
&& other.blockState.getSerialString().equals(this.blockState.getSerialString());
}
@Override
public String toString() { return this.serialize(); }
//=================//
// (de)serializing //
//=================//
public String serialize()
{
if (this.serialString == null)
{
this.serialString = this.biome.getSerialString() + FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString();
}
return this.serialString;
}
public static BlockBiomeWrapperPair deserialize(String str, ILevelWrapper levelWrapper) throws DataCorruptedException
{
int separatorIndex = str.indexOf(FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING);
if (separatorIndex == -1)
{
throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry ["+str+"], unable to find separator.");
}
IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapperOrGetDefault(str.substring(0, separatorIndex), levelWrapper);
IBlockStateWrapper blockState = WRAPPER_FACTORY.deserializeBlockStateWrapperOrGetDefault(str.substring(separatorIndex+FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING.length()), levelWrapper);
return BlockBiomeWrapperPair.get(blockState, biome);
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java
+52 -190
View File
@@ -19,7 +19,7 @@
package com.seibel.distanthorizons.core.dataObjects.fullData;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.dataObjects.BlockBiomeWrapperPair;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.LodUtil;
@@ -28,9 +28,7 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStrea
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import org.apache.logging.log4j.LogManager;
import com.seibel.distanthorizons.core.logging.DhLogger;
import java.io.*;
@@ -60,15 +58,15 @@ public class FullDataPointIdMap
*/
private static final boolean RUN_SERIALIZATION_DUPLICATE_VALIDATION = false;
/** Distant Horizons - Block State Wrapper */
private static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_";
public static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_";
/** should only be used for debugging */
private long pos;
/** The index should be the same as the Entry's ID */
private final ArrayList<Entry> entryList = new ArrayList<>();
private final ConcurrentHashMap<Entry, Integer> idMap = new ConcurrentHashMap<>();
/** The index should be the same as the BlockBiomeWrapperPair's ID */
private final ArrayList<BlockBiomeWrapperPair> blockBiomePairList = new ArrayList<>();
private final ConcurrentHashMap<BlockBiomeWrapperPair, Integer> idMap = new ConcurrentHashMap<>();
private int cachedHashCode = 0;
@@ -90,28 +88,28 @@ public class FullDataPointIdMap
public IBiomeWrapper getBiomeWrapper(int id) throws IndexOutOfBoundsException { return this.getEntry(id).biome; }
/** @see FullDataPointIdMap#getEntry(int) */
public IBlockStateWrapper getBlockStateWrapper(int id) throws IndexOutOfBoundsException { return this.getEntry(id).blockState; }
/** @throws IndexOutOfBoundsException if the given ID isn't in the {@link FullDataPointIdMap#entryList} */
private Entry getEntry(int id) throws IndexOutOfBoundsException
/** @throws IndexOutOfBoundsException if the given ID isn't in the {@link FullDataPointIdMap#blockBiomePairList} */
private BlockBiomeWrapperPair getEntry(int id) throws IndexOutOfBoundsException
{
Entry entry;
BlockBiomeWrapperPair pair;
try
{
entry = this.entryList.get(id);
pair = this.blockBiomePairList.get(id);
}
catch (IndexOutOfBoundsException e)
{
throw new IndexOutOfBoundsException("FullData ID Map out of sync for pos: "+DhSectionPos.toString(this.pos)+". ID: ["+id+"] greater than the number of known ID's: ["+this.entryList.size()+"].");
throw new IndexOutOfBoundsException("FullData ID Map out of sync for pos: "+DhSectionPos.toString(this.pos)+". ID: ["+id+"] greater than the number of known ID's: ["+this.blockBiomePairList.size()+"].");
}
return entry;
return pair;
}
/** @return -1 if the list is empty */
public int getMaxValidId() { return this.entryList.size() - 1; }
public int size() { return this.entryList.size(); }
public int getMaxValidId() { return this.blockBiomePairList.size() - 1; }
public int size() { return this.blockBiomePairList.size(); }
public boolean isEmpty() { return this.entryList.isEmpty(); }
public boolean isEmpty() { return this.blockBiomePairList.isEmpty(); }
public long getPos() { return this.pos; }
@@ -125,11 +123,11 @@ public class FullDataPointIdMap
* If an entry with the given values already exists nothing will
* be added but the existing item's ID will still be returned.
*/
public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(Entry.getEntry(biome, blockState)); }
private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry)
public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(BlockBiomeWrapperPair.get(blockState, biome)); }
private int addIfNotPresentAndGetId(BlockBiomeWrapperPair pair)
{
// try getting the existing ID
Integer nullableId = this.idMap.get(biomeBlockStateEntry);
Integer nullableId = this.idMap.get(pair);
if (nullableId != null)
{
return nullableId;
@@ -137,7 +135,7 @@ public class FullDataPointIdMap
// create the new ID
return this.idMap.compute(biomeBlockStateEntry, (Entry newBiomeBlockStateEntry, Integer currentId) ->
return this.idMap.compute(pair, (BlockBiomeWrapperPair newPair, Integer currentId) ->
{
if (currentId != null)
{
@@ -146,8 +144,8 @@ public class FullDataPointIdMap
// Add the new ID
currentId = this.entryList.size();
this.entryList.add(biomeBlockStateEntry);
currentId = this.blockBiomePairList.size();
this.blockBiomePairList.add(newPair);
// invalidate the cached hash code
this.cachedHashCode = 0;
@@ -157,7 +155,7 @@ public class FullDataPointIdMap
}
/**
* Adds every {@link Entry} from inputMap into this map. <br>
* Adds every {@link BlockBiomeWrapperPair} from inputMap into this map. <br>
* Allows duplicate entries. <br><br>
*
* Allowing duplicate entries should be done if a datasource is just being read in and
@@ -167,19 +165,19 @@ public class FullDataPointIdMap
*/
public void addAll(FullDataPointIdMap inputMap)
{
ArrayList<Entry> entriesToMerge = inputMap.entryList;
for (int i = 0; i < entriesToMerge.size(); i++)
ArrayList<BlockBiomeWrapperPair> pairsToMerge = inputMap.blockBiomePairList;
for (int i = 0; i < pairsToMerge.size(); i++)
{
Entry entity = entriesToMerge.get(i);
this.add(entity);
BlockBiomeWrapperPair pair = pairsToMerge.get(i);
this.add(pair);
}
}
/** allows for adding duplicate {@link Entry} */
private void add(Entry biomeBlockStateEntry)
/** allows for adding duplicate {@link BlockBiomeWrapperPair} */
private void add(BlockBiomeWrapperPair pair)
{
int id = this.entryList.size();
this.entryList.add(biomeBlockStateEntry);
this.idMap.put(biomeBlockStateEntry, id);
int id = this.blockBiomePairList.size();
this.blockBiomePairList.add(pair);
this.idMap.put(pair, id);
// invalidate the cached hash code
this.cachedHashCode = 0;
@@ -196,23 +194,23 @@ public class FullDataPointIdMap
*/
public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap inputMap)
{
ArrayList<Entry> entriesToMerge = inputMap.entryList;
int[] remappedEntryIds = new int[entriesToMerge.size()];
ArrayList<BlockBiomeWrapperPair> entriesToMerge = inputMap.blockBiomePairList;
int[] remappedPairIds = new int[entriesToMerge.size()];
for (int i = 0; i < entriesToMerge.size(); i++)
{
Entry entity = entriesToMerge.get(i);
BlockBiomeWrapperPair entity = entriesToMerge.get(i);
int id = this.addIfNotPresentAndGetId(entity);
remappedEntryIds[i] = id;
remappedPairIds[i] = id;
}
return remappedEntryIds;
return remappedPairIds;
}
/** Should only be used if this map is going to be reused, otherwise bad things will happen. */
public void clear(long pos)
{
this.pos = pos;
this.entryList.clear();
this.blockBiomePairList.clear();
this.idMap.clear();
this.cachedHashCode = 0;
}
@@ -226,27 +224,27 @@ public class FullDataPointIdMap
/** Serializes all contained entries into the given stream, formatted in UTF */
public void serialize(DhDataOutputStream outputStream) throws IOException
{
outputStream.writeInt(this.entryList.size());
outputStream.writeInt(this.blockBiomePairList.size());
// only used when debugging
HashMap<String, FullDataPointIdMap.Entry> dataPointEntryBySerialization = new HashMap<>();
HashMap<String, BlockBiomeWrapperPair> dataPointEntryBySerialization = new HashMap<>();
for (Entry entry : this.entryList)
for (BlockBiomeWrapperPair pair : this.blockBiomePairList)
{
String entryString = entry.serialize();
String entryString = pair.serialize();
outputStream.writeUTF(entryString);
if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
{
if (dataPointEntryBySerialization.containsKey(entryString))
{
LOGGER.error("Duplicate serialized entry found with serial: " + entryString);
LOGGER.error("Duplicate serialized pair found with serial: " + entryString);
}
if (dataPointEntryBySerialization.containsValue(entry))
if (dataPointEntryBySerialization.containsValue(pair))
{
LOGGER.error("Duplicate serialized entry found with value: " + entry.serialize());
LOGGER.error("Duplicate serialized pair found with value: " + pair.serialize());
}
dataPointEntryBySerialization.put(entryString, entry);
dataPointEntryBySerialization.put(entryString, pair);
}
}
}
@@ -262,7 +260,7 @@ public class FullDataPointIdMap
// only used when debugging
HashMap<String, FullDataPointIdMap.Entry> dataPointEntryBySerialization = new HashMap<>();
HashMap<String, BlockBiomeWrapperPair> dataPointEntryBySerialization = new HashMap<>();
FullDataPointIdMap newMap = new FullDataPointIdMap(pos);
for (int i = 0; i < entityCount; i++)
@@ -275,8 +273,8 @@ public class FullDataPointIdMap
String entryString = inputStream.readUTF();
Entry newEntry = Entry.deserialize(entryString, levelWrapper);
newMap.entryList.add(newEntry);
BlockBiomeWrapperPair newPair = BlockBiomeWrapperPair.deserialize(entryString, levelWrapper);
newMap.blockBiomePairList.add(newPair);
if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
{
@@ -284,11 +282,11 @@ public class FullDataPointIdMap
{
LOGGER.error("Duplicate deserialized entry found with serial: " + entryString);
}
if (dataPointEntryBySerialization.containsValue(newEntry))
if (dataPointEntryBySerialization.containsValue(newPair))
{
LOGGER.error("Duplicate deserialized entry found with value: " + newEntry.serialize());
LOGGER.error("Duplicate deserialized entry found with value: " + newPair.serialize());
}
dataPointEntryBySerialization.put(entryString, newEntry);
dataPointEntryBySerialization.put(entryString, newPair);
}
}
@@ -334,149 +332,13 @@ public class FullDataPointIdMap
private void generateHashCode()
{
int result = DhSectionPos.hashCode(this.pos);
for (int i = 0; i < this.entryList.size(); i++)
for (int i = 0; i < this.blockBiomePairList.size(); i++)
{
result = 31 * result + this.entryList.hashCode();
result = 31 * result + this.blockBiomePairList.hashCode();
}
this.cachedHashCode = result;
}
//==============//
// helper class //
//==============//
private static final class Entry
{
private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class);
/** two levels are present so we don't need to use a key object */
private static final ConcurrentHashMap<IBiomeWrapper, ConcurrentHashMap<IBlockStateWrapper, Entry>> ENTRY_BY_BLOCKSTATE_BY_BIOMEWRAPPER = new ConcurrentHashMap<>();
public final IBiomeWrapper biome;
public final IBlockStateWrapper blockState;
private int hashCode = 0;
private boolean hashGenerated = false;
private String serialString = null;
//=============//
// constructor //
//=============//
public static Entry getEntry(IBiomeWrapper biome, IBlockStateWrapper blockState)
{
// check for existing entry
ConcurrentHashMap<IBlockStateWrapper, Entry> entryByBlockState = ENTRY_BY_BLOCKSTATE_BY_BIOMEWRAPPER.get(biome);
if (entryByBlockState != null)
{
Entry entry = entryByBlockState.get(blockState);
if (entry != null)
{
return entry;
}
}
// Lazily create the inner map and new Entry
return ENTRY_BY_BLOCKSTATE_BY_BIOMEWRAPPER
.computeIfAbsent(biome, newBiome -> new ConcurrentHashMap<>())
.computeIfAbsent(blockState, newBlockState -> new Entry(biome, blockState));
}
private Entry(IBiomeWrapper biome, IBlockStateWrapper blockState)
{
this.biome = biome;
this.blockState = blockState;
}
//===========//
// overrides //
//===========//
/**
* Reminder: this hash code won't always be unique, collisions can occur;
* because of that this hash shouldn't be the only unique identifier for this object.
*/
@Override
public int hashCode()
{
// cache the hash code to improve speed
if (!this.hashGenerated)
{
this.hashCode = generateHashCode(this);
this.hashGenerated = true;
}
return this.hashCode;
}
private static int generateHashCode(Entry entry) { return generateHashCode(entry.biome, entry.blockState); }
private static int generateHashCode(IBiomeWrapper biome, IBlockStateWrapper blockState)
{
final int prime = 31;
int result = 1;
// the biome and blockstate hashcode should be already calculated by the time
// we get here, so this operation should be very fast
result = prime * result + (biome == null ? 0 : biome.hashCode());
result = prime * result + (blockState == null ? 0 : blockState.hashCode());
return result;
}
@Override
public boolean equals(Object otherObj)
{
if (otherObj == this)
{
return true;
}
if (!(otherObj instanceof Entry))
{
return false;
}
Entry other = (Entry) otherObj;
return other.biome.getSerialString().equals(this.biome.getSerialString())
&& other.blockState.getSerialString().equals(this.blockState.getSerialString());
}
@Override
public String toString() { return this.serialize(); }
//=================//
// (de)serializing //
//=================//
public String serialize()
{
if (this.serialString == null)
{
this.serialString = this.biome.getSerialString() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString();
}
return this.serialString;
}
public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws DataCorruptedException
{
int separatorIndex = str.indexOf(BLOCK_STATE_SEPARATOR_STRING);
if (separatorIndex == -1)
{
throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry ["+str+"], unable to find separator.");
}
IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapperOrGetDefault(str.substring(0, separatorIndex), levelWrapper);
IBlockStateWrapper blockState = WRAPPER_FACTORY.deserializeBlockStateWrapperOrGetDefault(str.substring(separatorIndex+BLOCK_STATE_SEPARATOR_STRING.length()), levelWrapper);
return Entry.getEntry(biome, blockState);
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java
+1 -28
View File
@@ -20,7 +20,6 @@
package com.seibel.distanthorizons.core.dataObjects.fullData.sources;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -34,7 +33,6 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import com.seibel.distanthorizons.core.logging.DhLogger;
import java.io.*;
import java.util.Arrays;
@@ -48,7 +46,7 @@ import java.util.Arrays;
* @see FullDataPointUtil
* @see FullDataSourceV2
*/
public class FullDataSourceV1 implements IDataSource<IDhLevel>
public class FullDataSourceV1
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
@@ -95,28 +93,13 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
}
//======//
// data //
//======//
@Deprecated
@Override
public boolean update(FullDataSourceV2 dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); }
//=====================//
// setters and getters //
//=====================//
@Override
public Long getKey() { return this.pos; }
@Override
public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
@Override
public long getPos() { return this.pos; }
public void resizeDataStructuresForRepopulation(long pos)
@@ -125,7 +108,6 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
this.pos = pos;
}
@Override
public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - SECTION_SIZE_OFFSET); }
public boolean isEmpty() { return this.isEmpty; }
@@ -378,15 +360,6 @@ public class FullDataSourceV1 implements IDataSource<IDhLevel>
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
//==================//
// override methods //
//==================//
@Override
public void close()
{ /* not currently needed */ }
//================//
// helper classes //
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java
+49 -36
View File
@@ -27,9 +27,8 @@ import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataOcclusionCuller;
import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
@@ -38,6 +37,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos;
import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil;
import com.seibel.distanthorizons.core.util.*;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
@@ -45,7 +45,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.coreapi.ModInfo;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import com.seibel.distanthorizons.core.logging.DhLogger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -61,7 +60,7 @@ import java.util.List;
*/
public class FullDataSourceV2
extends AbstractPhantomArrayList
implements IDataSource<IDhLevel>, IDhApiFullDataSource
implements IDhApiFullDataSource
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
/** useful for debugging, but can slow down update operations quite a bit due to being called so often. */
@@ -78,8 +77,6 @@ public class FullDataSourceV2
/** how many chunks wide this datasource is at detail level 0. */
public static final int NUMB_OF_CHUNKS_WIDE = WIDTH / LodUtil.CHUNK_WIDTH;
public static final byte DATA_FORMAT_VERSION = 1;
public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("FullDataV2");
@@ -87,10 +84,6 @@ public class FullDataSourceV2
private int cachedHashCode = 0;
private final long pos;
@Override
public Long getKey() { return this.pos; }
@Override
public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
public final FullDataPointIdMap mapping;
@@ -116,9 +109,7 @@ public class FullDataSourceV2
/**
* stored x/z, y <br>
* The y data should be sorted from top to bottom <br>
* TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately
* there's something in the render data logic that expects this order so we can't change it right now
* The y data should be sorted from top to bottom
*/
public final LongArrayList[] dataPoints;
@@ -228,7 +219,7 @@ public class FullDataSourceV2
private FullDataSourceV2(
long pos,
FullDataPointIdMap mapping, @Nullable LongArrayList[] data,
@Nullable byte[] columnGenerationSteps, @Nullable byte[] columnWorldCompressionMode,
byte @Nullable [] columnGenerationSteps, byte @Nullable [] columnWorldCompressionMode,
boolean empty)
{
super(ARRAY_LIST_POOL, 2, 0, WIDTH * WIDTH);
@@ -289,11 +280,11 @@ public class FullDataSourceV2
// getters //
//=========//
public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException
public LongArrayList getColumnAtRelPos(int relX, int relZ) throws IndexOutOfBoundsException
{ return this.dataPoints[relativePosToIndex(relX, relZ)]; }
@Nullable
public LongArrayList tryGet(int relX, int relZ)
public LongArrayList tryGetColumnAtRelPos(int relX, int relZ)
{
int index = tryGetRelativePosToIndex(relX, relZ);
if (index == -1)
@@ -308,7 +299,7 @@ public class FullDataSourceV2
* returns {@link FullDataPointUtil#EMPTY_DATA_POINT} if the given {@link DhBlockPos}
* is outside this data source's boundaries.
*/
public long getAtBlockPos(DhBlockPos blockPos)
public long getDataPointAtBlockPos(DhBlockPos blockPos)
{
DhLodPos requestedPos = new DhLodPos(LodUtil.BLOCK_DETAIL_LEVEL, blockPos.getX(), blockPos.getZ());
@@ -332,7 +323,7 @@ public class FullDataSourceV2
DhLodPos relativePos = requestedPos.getDhSectionRelativePositionForDetailLevel(requestDetailLevel);
// get the data column
LongArrayList dataColumn = this.get(relativePos.x, relativePos.z);
LongArrayList dataColumn = this.getColumnAtRelPos(relativePos.x, relativePos.z);
if (dataColumn == null)
{
return FullDataPointUtil.EMPTY_DATA_POINT;
@@ -376,9 +367,7 @@ public class FullDataSourceV2
// updating //
//==========//
@Override
public boolean update(@NotNull FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); }
public boolean update(@NotNull FullDataSourceV2 inputDataSource)
public boolean updateFromChunk(@NotNull FullDataSourceV2 inputDataSource)
{
// don't try updating if the input is empty
if (inputDataSource.mapping.isEmpty())
@@ -406,7 +395,7 @@ public class FullDataSourceV2
// copy over application flag if either are set to continue propagating
(BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent))
// don't propagate past the top of the tree
&& (DhSectionPos.getDetailLevel(this.pos) < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL);
&& (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.ROOT_SECTION_DETAIL_LEVEL);
}
// null check to prevent setting a flag we don't want to save in the DB
@@ -415,7 +404,7 @@ public class FullDataSourceV2
this.applyToChildren =
(BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren))
// don't propagate past the bottom of the tree
&& (DhSectionPos.getDetailLevel(this.pos) > AbstractDataSourceHandler.MIN_SECTION_DETAIL_LEVEL);
&& (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.LEAF_SECTION_DETAIL_LEVEL);
}
}
else if (inputDetailLevel + 1 == thisDetailLevel)
@@ -426,7 +415,7 @@ public class FullDataSourceV2
this.applyToParent =
dataChanged
&& (BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent))
&& (DhSectionPos.getDetailLevel(this.pos) < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL);
&& (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.ROOT_SECTION_DETAIL_LEVEL);
}
else if (inputDetailLevel - 1 == thisDetailLevel)
@@ -438,7 +427,7 @@ public class FullDataSourceV2
this.applyToChildren =
dataChanged
&& (BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren))
&& (DhSectionPos.getDetailLevel(this.pos) > AbstractDataSourceHandler.MIN_SECTION_DETAIL_LEVEL);
&& (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.LEAF_SECTION_DETAIL_LEVEL);
}
else
{
@@ -463,7 +452,7 @@ public class FullDataSourceV2
{
for (int z = 0; z < WIDTH; z++)
{
LongArrayList dataColumn = this.get(x, z);
LongArrayList dataColumn = this.getColumnAtRelPos(x, z);
if (dataColumn != null
&& dataColumn.size() > 1)
{
@@ -1105,6 +1094,38 @@ public class FullDataSourceV2
//===================//
// adjacent clearing //
//===================//
/** Removes any non-adjacent data from the given direction. */
public void clearAllNonAdjData(EDhDirection direction)
{
long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction);
int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos);
int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos);
int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos);
int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos);
for (int relX = 0; relX < FullDataSourceV2.WIDTH; relX++)
{
for (int relZ = 0; relZ < FullDataSourceV2.WIDTH; relZ++)
{
// skip non-adjacent data
if (relX >= minX && relX < maxX
&& relZ >= minZ && relZ < maxZ)
{
continue;
}
LongArrayList dataColumn = this.getColumnAtRelPos(relX, relZ);
dataColumn.clear();
dataColumn.add(FullDataPointUtil.EMPTY_DATA_POINT);
}
}
}
//================//
// helper methods //
//================//
@@ -1205,18 +1226,10 @@ public class FullDataSourceV2
// setters and getters //
//=====================//
@Override
public long getPos() { return this.pos; }
@Override
public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); }
public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ)
{
int index = relativePosToIndex(relX, relZ);
return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps.getByte(index));
}
public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep, EDhApiWorldCompressionMode worldCompressionMode)
{
int index = relativePosToIndex(relX, relZ);
@@ -1280,7 +1293,7 @@ public class FullDataSourceV2
@Override
public List<DhApiTerrainDataPoint> getApiDataPointColumn(int relX, int relZ) throws IndexOutOfBoundsException
{
LongArrayList dataColumn = this.get(relX, relZ);
LongArrayList dataColumn = this.getColumnAtRelPos(relX, relZ);
ArrayList<DhApiTerrainDataPoint> apiList = new ArrayList<>();
for (int i = 0; i < dataColumn.size(); i++)
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java
-96
View File
@@ -1,96 +0,0 @@
package com.seibel.distanthorizons.core.dataObjects.render;
import com.google.common.cache.Cache;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
/**
* Wrapper for {@link ColumnRenderSource} that handles reference counting
* and cache tracking.
*/
public class CachedColumnRenderSource implements AutoCloseable
{
/** an externally handled future that will complete once the {@link CachedColumnRenderSource#columnRenderSource} has finished loading */
public final CompletableFuture<CachedColumnRenderSource> loadFuture;
/** will be null initially, should be non-null once the corresponding load future is done */
@Nullable
public ColumnRenderSource columnRenderSource = null;
private final AtomicInteger referenceCount;
private final Cache<Long, CachedColumnRenderSource> cachedRenderSourceByPos;
private final ReentrantLock getterLock;
//=============//
// constructor //
//=============//
public CachedColumnRenderSource(
@NotNull CompletableFuture<CachedColumnRenderSource> loadFuture,
@NotNull ReentrantLock getterLock,
@NotNull Cache<Long, CachedColumnRenderSource> cachedRenderSourceByPos)
{
this.loadFuture = loadFuture;
this.getterLock = getterLock;
this.referenceCount = new AtomicInteger(1);
this.cachedRenderSourceByPos = cachedRenderSourceByPos;
}
//====================//
// reference counting //
//====================//
public void markInUse() { this.referenceCount.getAndIncrement(); }
//================//
// base overrides //
//================//
/**
* Will be called multiple times,
* however it will only close the underlying data once
* all references have closed.
*/
@Override
public void close() throws IllegalStateException
{
try
{
// lock to prevent other threads for accessing the cache if we invalidate it
this.getterLock.lock();
// should only happen if something goes wrong up-stream
if (this.columnRenderSource == null)
{
return;
}
// only close once everyone is done with this datasource
int refCount = this.referenceCount.decrementAndGet();
if (refCount == 0)
{
this.cachedRenderSourceByPos.invalidate(this.columnRenderSource.pos);
this.columnRenderSource.close();
}
else if (refCount < 0)
{
throw new IllegalStateException("Render source ["+this.columnRenderSource.pos+"] reference count incorrect. Object already closed.");
}
}
finally
{
this.getterLock.unlock();
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java
+16 -25
View File
@@ -23,17 +23,13 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.coreapi.ModInfo;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnQuadView;
import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import com.seibel.distanthorizons.core.util.ColorUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import com.seibel.distanthorizons.core.logging.DhLogger;
import java.util.concurrent.atomic.AtomicLong;
/**
* Stores the render data used to generate OpenGL buffers.
*
@@ -43,10 +39,8 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
public static final boolean DO_SAFETY_CHECKS = ModInfo.IS_DEV_BUILD;
public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
/** width of this data in columns */
public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); // 64
/** measured in data columns */
public static final int WIDTH = 64;
public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Render Source");
@@ -63,8 +57,6 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
private boolean isEmpty = true;
public AtomicLong localVersion = new AtomicLong(0); // used to track changes to the data source, so that buffers can be updated when necessary
//==============//
@@ -88,9 +80,9 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
this.verticalDataCount = maxVerticalSize;
this.renderDataContainer = this.pooledArraysCheckout.getLongArray(0, SECTION_SIZE * SECTION_SIZE * this.verticalDataCount);
this.renderDataContainer = this.pooledArraysCheckout.getLongArray(0, WIDTH * WIDTH * this.verticalDataCount);
this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE];
this.debugSourceFlags = new DebugSourceFlag[WIDTH * WIDTH];
}
@@ -99,19 +91,19 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
// datapoint manipulation //
//========================//
public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer.getLong(posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex); }
public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer.getLong(posX * WIDTH * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex); }
public ColumnArrayView getVerticalDataPointView(int posX, int posZ)
{
int offset = posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount;
int offset = posX * WIDTH * this.verticalDataCount + posZ * this.verticalDataCount;
// don't allow returning views that are outside this render source's bounds
if (offset >= this.renderDataContainer.size())
{
return null;
}
else if (posX < 0 || posX >= SECTION_SIZE
|| posZ < 0 || posZ >= SECTION_SIZE)
else if (posX < 0 || posX >= WIDTH
|| posZ < 0 || posZ >= WIDTH)
{
return null;
}
@@ -120,8 +112,8 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
offset, this.verticalDataCount);
}
public ColumnQuadView getFullQuadView() { return this.getQuadViewOverRange(0, 0, SECTION_SIZE, SECTION_SIZE); }
public ColumnQuadView getQuadViewOverRange(int quadX, int quadZ, int quadXSize, int quadZSize) { return new ColumnQuadView(this.renderDataContainer, SECTION_SIZE, this.verticalDataCount, quadX, quadZ, quadXSize, quadZSize); }
public ColumnQuadView getFullQuadView() { return this.getQuadViewOverRange(0, 0, WIDTH, WIDTH); }
public ColumnQuadView getQuadViewOverRange(int quadX, int quadZ, int quadXSize, int quadZSize) { return new ColumnQuadView(this.renderDataContainer, WIDTH, this.verticalDataCount, quadX, quadZ, quadXSize, quadZSize); }
@@ -131,9 +123,8 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
public Long getPos() { return this.pos; }
public Long getKey() { return this.pos; }
public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); }
public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - SECTION_SIZE_OFFSET); }
public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); }
public boolean isEmpty() { return this.isEmpty; }
public void markNotEmpty() { this.isEmpty = false; }
@@ -147,15 +138,15 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
}
for (int x = 0; x < SECTION_SIZE; x++)
for (int x = 0; x < WIDTH; x++)
{
for (int z = 0; z < SECTION_SIZE; z++)
for (int z = 0; z < WIDTH; z++)
{
ColumnArrayView columnArrayView = this.getVerticalDataPointView(x,z);
for (int i = 0; i < columnArrayView.size; i++)
{
long dataPoint = columnArrayView.get(i);
if (!RenderDataPointUtil.isVoid(dataPoint))
if (!RenderDataPointUtil.hasZeroHeight(dataPoint))
{
return true;
}
@@ -179,12 +170,12 @@ public class ColumnRenderSource extends AbstractPhantomArrayList
{
for (int z = zStart; z < zStart + zWidth; z++)
{
this.debugSourceFlags[x * SECTION_SIZE + z] = flag;
this.debugSourceFlags[x * WIDTH + z] = flag;
}
}
}
public DebugSourceFlag debugGetFlag(int ox, int oz) { return this.debugSourceFlags[ox * SECTION_SIZE + oz]; }
public DebugSourceFlag debugGetFlag(int ox, int oz) { return this.debugSourceFlags[ox * WIDTH + oz]; }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java
+6 -6
View File
@@ -35,7 +35,7 @@ public final class BufferQuad
public static final int NORMAL_MAX_QUAD_WIDTH = 2048;
/**
* The maximum number of blocks wide a quad can be
* when {@link Config.Client.Advanced.Graphics.AdvancedGraphics#earthCurveRatio earthCurveRatio}
* when {@link Config.Client.Advanced.Graphics.Experimental#earthCurveRatio earthCurveRatio}
* is enabled.
*/
public static final int MAX_QUAD_WIDTH_FOR_EARTH_CURVATURE = LodUtil.CHUNK_WIDTH;
@@ -99,7 +99,7 @@ public final class BufferQuad
if (compareDirection == BufferMergeDirectionEnum.EastWest)
{
switch (this.direction.getAxis())
switch (this.direction.axis)
{
case X:
return threeDimensionalCompare(this.x, this.y, this.z, quad.x, quad.y, quad.z);
@@ -109,12 +109,12 @@ public final class BufferQuad
return threeDimensionalCompare(this.z, this.y, this.x, quad.z, quad.y, quad.x);
default:
throw new IllegalArgumentException("Invalid Axis enum: " + this.direction.getAxis());
throw new IllegalArgumentException("Invalid Axis enum: [" + this.direction.axis + "].");
}
}
else
{
switch (this.direction.getAxis())
switch (this.direction.axis)
{
case X:
return threeDimensionalCompare(this.x, this.z, this.y, quad.x, quad.z, quad.y);
@@ -124,7 +124,7 @@ public final class BufferQuad
return threeDimensionalCompare(this.z, this.x, this.y, quad.z, quad.x, quad.y);
default:
throw new IllegalArgumentException("Invalid Axis enum: " + this.direction.getAxis());
throw new IllegalArgumentException("Invalid Axis enum: [" + this.direction.axis + "].");
}
}
}
@@ -169,7 +169,7 @@ public final class BufferQuad
short thisParallelCompareStartPos; // edge parallel to the merge direction
short otherPerpendicularCompareStartPos;
short otherParallelCompareStartPos;
switch (this.direction.getAxis())
switch (this.direction.axis)
{
default: // shouldn't normally happen, just here to make the compiler happy
case X:
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java
+273 -202
View File
@@ -23,38 +23,27 @@ import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.util.ColorUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
import com.seibel.distanthorizons.core.render.renderer.LodRenderer;
import com.seibel.distanthorizons.coreapi.util.MathUtil;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.jetbrains.annotations.NotNull;
import java.util.Arrays;
public class ColumnBox
{
private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
/**
* if the skylight has this value that means
* no data is expected
*/
private static final byte SKYLIGHT_EMPTY = -1;
/**
* if the skylight has this value that means
* that block position is covered/occuled by an adjacent block/column.
* that block position is covered/occluded by an adjacent block/column.
*/
private static final byte SKYLIGHT_COVERED = -2;
private static final byte SKYLIGHT_COVERED = -1;
private static final ThreadLocal<byte[]> THREAD_LOCAL_SKY_LIGHT_ARRAY = ThreadLocal.withInitial(() ->
{
byte[] array = new byte[RenderDataPointUtil.MAX_WORLD_Y_SIZE];
Arrays.fill(array, SKYLIGHT_EMPTY);
return array;
});
@@ -63,8 +52,8 @@ public class ColumnBox
//=========//
public static void addBoxQuadsToBuilder(
LodQuadBuilder builder, IDhClientLevel clientLevel,
short xSize, short ySize, short zSize,
LodQuadBuilder builder, PhantomArrayListCheckout phantomArrayCheckout, IDhClientLevel clientLevel,
short width, short yHeight,
short minX, short minY, short minZ,
int color, byte irisBlockMaterialId, byte skyLight, byte blockLight,
long topData, long bottomData, ColumnArrayView[] adjData, boolean[] isAdjDataSameDetailLevel)
@@ -73,9 +62,9 @@ public class ColumnBox
// variable setup //
//================//
short maxX = (short) (minX + xSize);
short maxY = (short) (minY + ySize);
short maxZ = (short) (minZ + zSize);
short maxX = (short) (minX + width);
short maxY = (short) (minY + yHeight);
short maxZ = (short) (minZ + width);
byte skyLightTop = skyLight;
byte skyLightBot = RenderDataPointUtil.doesDataPointExist(bottomData) ? RenderDataPointUtil.getLightSky(bottomData) : 0;
@@ -111,15 +100,15 @@ public class ColumnBox
if (!isTransparent && isTopTransparent && RenderDataPointUtil.doesDataPointExist(topData))
{
skyLightTop = (byte) MathUtil.clamp(0, 15 - (RenderDataPointUtil.getYMax(topData) - minY), 15);
ySize = (short) (RenderDataPointUtil.getYMax(topData) - minY - 1);
yHeight = (short) (RenderDataPointUtil.getYMax(topData) - minY - 1);
}
else if (isTransparent && !isBottomTransparent && RenderDataPointUtil.doesDataPointExist(bottomData))
{
minY = (short) (minY + ySize - 1);
ySize = 1;
minY = (short) (minY + yHeight - 1);
yHeight = 1;
}
maxY = (short) (minY + ySize);
maxY = (short) (minY + yHeight);
}
@@ -128,16 +117,26 @@ public class ColumnBox
// add top and bottom faces //
//==========================//
boolean skipTop = RenderDataPointUtil.doesDataPointExist(topData) && (RenderDataPointUtil.getYMin(topData) == maxY) && !isTopTransparent;
if (!skipTop)
// top face
{
builder.addQuadUp(minX, maxY, minZ, xSize, zSize, ColorUtil.applyShade(color, MC.getShade(EDhDirection.UP)), irisBlockMaterialId, skyLightTop, blockLight);
boolean skipTop = RenderDataPointUtil.doesDataPointExist(topData)
&& (RenderDataPointUtil.getYMin(topData) == maxY)
&& !isTopTransparent;
if (!skipTop)
{
builder.addQuadUp(minX, maxY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.UP)), irisBlockMaterialId, skyLightTop, blockLight);
}
}
boolean skipBottom = RenderDataPointUtil.doesDataPointExist(bottomData) && (RenderDataPointUtil.getYMax(bottomData) == minY) && !isBottomTransparent;
if (!skipBottom)
// bottom face
{
builder.addQuadDown(minX, minY, minZ, xSize, zSize, ColorUtil.applyShade(color, MC.getShade(EDhDirection.DOWN)), irisBlockMaterialId, skyLightBot, blockLight);
boolean skipBottom = RenderDataPointUtil.doesDataPointExist(bottomData)
&& (RenderDataPointUtil.getYMax(bottomData) == minY)
&& !isBottomTransparent;
if (!skipBottom)
{
builder.addQuadDown(minX, minY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.DOWN)), irisBlockMaterialId, skyLightBot, blockLight);
}
}
@@ -148,84 +147,119 @@ public class ColumnBox
// NORTH face
{
ColumnArrayView adjCol = adjData[EDhDirection.NORTH.ordinal() - 2]; // TODO can we use something other than ordinal-2?
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.NORTH.ordinal() - 2];
ColumnArrayView adjCol = adjData[EDhDirection.NORTH.compassIndex];
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.NORTH.compassIndex];
// if the adjacent column is null that generally means the adjacent area hasn't been generated yet
if (adjCol == null)
{
// Add an adjacent face if this is opaque face or transparent over the void.
if (!isTransparent || overVoid)
{
builder.addQuadAdj(EDhDirection.NORTH, minX, minY, minZ, xSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
builder.addQuadAdj(
EDhDirection.NORTH,
minX, minY, minZ,
width, yHeight,
color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
}
}
else
{
makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.NORTH, minX, minY, minZ, xSize, ySize,
makeAdjVerticalQuad(
builder, phantomArrayCheckout,
adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.NORTH,
minX, minY, minZ, width, yHeight,
color, irisBlockMaterialId, blockLight);
}
}
// SOUTH face
{
ColumnArrayView adjCol = adjData[EDhDirection.SOUTH.ordinal() - 2];
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.SOUTH.ordinal() - 2];
ColumnArrayView adjCol = adjData[EDhDirection.SOUTH.compassIndex];
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.SOUTH.compassIndex];
if (adjCol == null)
{
if (!isTransparent || overVoid)
{
builder.addQuadAdj(EDhDirection.SOUTH, minX, minY, maxZ, xSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
builder.addQuadAdj(
EDhDirection.SOUTH,
minX, minY, maxZ,
width, yHeight,
color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
}
}
else
{
makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.SOUTH, minX, minY, maxZ, xSize, ySize,
makeAdjVerticalQuad(
builder, phantomArrayCheckout,
adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.SOUTH,
minX, minY, maxZ, width, yHeight,
color, irisBlockMaterialId, blockLight);
}
}
// WEST face
{
ColumnArrayView adjCol = adjData[EDhDirection.WEST.ordinal() - 2];
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.WEST.ordinal() - 2];
ColumnArrayView adjCol = adjData[EDhDirection.WEST.compassIndex];
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.WEST.compassIndex];
if (adjCol == null)
{
if (!isTransparent || overVoid)
{
builder.addQuadAdj(EDhDirection.WEST, minX, minY, minZ, zSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
builder.addQuadAdj(
EDhDirection.WEST,
minX, minY, minZ,
width, yHeight,
color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
}
}
else
{
makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.WEST, minX, minY, minZ, zSize, ySize,
makeAdjVerticalQuad(
builder, phantomArrayCheckout,
adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.WEST,
minX, minY, minZ, width, yHeight,
color, irisBlockMaterialId, blockLight);
}
}
// EAST face
{
ColumnArrayView adjCol = adjData[EDhDirection.EAST.ordinal() - 2];
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.EAST.ordinal() - 2];
ColumnArrayView adjCol = adjData[EDhDirection.EAST.compassIndex];
boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.EAST.compassIndex];
if (adjCol == null)
{
if (!isTransparent || overVoid)
{
builder.addQuadAdj(EDhDirection.EAST, maxX, minY, minZ, zSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
builder.addQuadAdj(
EDhDirection.EAST,
maxX, minY, minZ,
width, yHeight,
color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
}
}
else
{
makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.EAST, maxX, minY, minZ, zSize, ySize,
makeAdjVerticalQuad(
builder, phantomArrayCheckout,
adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.EAST,
maxX, minY, minZ, width, yHeight,
color, irisBlockMaterialId, blockLight);
}
}
}
private static void makeAdjVerticalQuad(
LodQuadBuilder builder, @NotNull ColumnArrayView adjColumnView, boolean adjacentIsSameDetailLevel, int caveCullingMaxY, EDhDirection direction,
LodQuadBuilder builder, PhantomArrayListCheckout phantomArrayCheckout,
@NotNull ColumnArrayView adjColumnView, boolean adjacentIsSameDetailLevel, int caveCullingMaxY, EDhDirection direction,
short x, short yMin, short z, short horizontalWidth, short ySize,
int color, byte irisBlockMaterialId, byte blockLight)
{
// pooled arrays
LongArrayList segments = phantomArrayCheckout.getLongArray(0, 0);
LongArrayList newSegments = phantomArrayCheckout.getLongArray(1, 0);
//==================//
// create face with //
// no adjacent data //
@@ -233,177 +267,164 @@ public class ColumnBox
color = ColorUtil.applyShade(color, MC.getShade(direction));
// if there isn't any data adjacent to this LOD,
// just add the full vertical quad
if (adjColumnView.size == 0 || RenderDataPointUtil.isVoid(adjColumnView.get(0)))
if (adjColumnView.size == 0
|| RenderDataPointUtil.hasZeroHeight(adjColumnView.get(0)))
{
builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight);
return;
}
//===========================//
// Determine face visibility //
// based on it's neighbors //
//===========================//
//=================================//
// determine face visibility/light //
//=================================//
boolean transparencyEnabled = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled;
boolean inputTransparent = ColorUtil.getAlpha(color) < 255 && transparencyEnabled;
short yMax = (short) (yMin + ySize);
short yMax = (short) (yMin + ySize); // min is inclusive, max is exclusive
byte[] skyLightAtInputPos = THREAD_LOCAL_SKY_LIGHT_ARRAY.get();
try
int adjCount = adjColumnView.size();
// Start with the entire range at max light
segments.add(YSegmentUtil.encode(yMin, yMax, LodUtil.MAX_MC_LIGHT));
// Process each adjacent datapoint and split segments as needed
for (int adjIndex = 0; adjIndex < adjCount; adjIndex++)
{
// set the initial sky-lights for this face,
// if nothing overlaps or overhangs the face should have max sky light
Arrays.fill(skyLightAtInputPos, yMin, yMax, LodUtil.MAX_MC_LIGHT);
long adjPoint = adjColumnView.get(adjIndex);
short adjMinY = RenderDataPointUtil.getYMin(adjPoint);
short adjMaxY = RenderDataPointUtil.getYMax(adjPoint);
// iterate top down
int adjCount = adjColumnView.size();
for (int adjIndex = 0; adjIndex < adjCount; adjIndex++)
// skip empty adjacent points
// or points below this one
if (!RenderDataPointUtil.doesDataPointExist(adjPoint)
|| RenderDataPointUtil.hasZeroHeight(adjPoint)
|| yMax <= adjMinY)
{
long adjPoint = adjColumnView.get(adjIndex);
short adjMinY = RenderDataPointUtil.getYMin(adjPoint);
short adjMaxY = RenderDataPointUtil.getYMax(adjPoint);
// skip empty adjacent datapoints
if (!RenderDataPointUtil.doesDataPointExist(adjPoint)
|| RenderDataPointUtil.isVoid(adjPoint))
{
continue;
}
// skip this adjacent datapoint if it's above the input datapoint (since it can't affect the input data point)
if (yMax <= adjMinY)
{
continue;
}
long adjAbovePoint = (adjIndex != 0) ? adjColumnView.get(adjIndex - 1) : RenderDataPointUtil.EMPTY_DATA;
long adjBelowPoint = (adjIndex + 1 < adjCount) ? adjColumnView.get(adjIndex + 1) : RenderDataPointUtil.EMPTY_DATA;
// if the adjacent data point is over the void
// don't consider it as transparent
boolean adjOverVoid = !RenderDataPointUtil.doesDataPointExist(adjBelowPoint);
boolean adjTransparent = !adjOverVoid
&& RenderDataPointUtil.getAlpha(adjPoint) < 255
&& transparencyEnabled;
//=================================//
// set sky light based on adjacent //
//=================================//
// set light based on overlapping adjacent
if (!adjTransparent)
{
// adj opaque
// mark positions adjacent is covering
byte adjSkyLight = RenderDataPointUtil.getLightSky(adjPoint);
for (int i = adjMinY; i < adjMaxY; i++)
{
byte skyLightAtPos = skyLightAtInputPos[i];
// if the adjacent is a different detail level, we want to render adjacent opaque
// faces to try and reduce the chance of holes on detail level borders
boolean adjacentCoversThis =
// if the adjacent is the same detail level, no special handling is necessary
!adjacentIsSameDetailLevel
// if the adjacent face is underground we probably don't need it
&& RenderDataPointUtil.getYMax(adjPoint) >= caveCullingMaxY
// check if this face is on a border
&&
(
(x == 0 && direction == EDhDirection.WEST)
|| (z == 0 && direction == EDhDirection.NORTH)
// TODO why does 256 represent a border? aren't LODs only 64 datapoints wide?
|| (x == 256 && direction == EDhDirection.EAST)
|| (z == 256 && direction == EDhDirection.SOUTH)
);
byte newSkyLightAtPos = adjacentCoversThis ? adjSkyLight : SKYLIGHT_COVERED;
skyLightAtInputPos[i] = (byte) Math.min(newSkyLightAtPos, skyLightAtPos);
}
}
else
{
// adjacent is transparent,
// use datapoint below adjacent for lighting
byte belowSkyLight = RenderDataPointUtil.getLightSky(adjBelowPoint);
for (int i = adjMinY; i < adjMaxY; i++)
{
byte skyLightAtPos = skyLightAtInputPos[i];
skyLightAtInputPos[i] = (byte) Math.min(belowSkyLight, skyLightAtPos);
}
}
// fill in sky light up to the next DP,
// this is done to handle overhangs
byte adjSkyLight = RenderDataPointUtil.getLightSky(adjPoint);
int adjAboveMinY = RenderDataPointUtil.getYMin(adjAbovePoint);
for (int i = adjMaxY; i < adjAboveMinY; i++)
{
byte skyLightAtPos = skyLightAtInputPos[i];
skyLightAtInputPos[i] = (byte) Math.min(adjSkyLight, skyLightAtPos);
}
continue;
}
long adjAbovePoint = (adjIndex != 0) ? adjColumnView.get(adjIndex - 1) : RenderDataPointUtil.EMPTY_DATA;
long adjBelowPoint = (adjIndex + 1 < adjCount) ? adjColumnView.get(adjIndex + 1) : RenderDataPointUtil.EMPTY_DATA;
//=======================//
// create vertical faces //
//=======================//
boolean adjOverVoid = !RenderDataPointUtil.doesDataPointExist(adjBelowPoint);
boolean adjTransparent = !adjOverVoid
&& RenderDataPointUtil.getAlpha(adjPoint) < 255
&& transparencyEnabled;
boolean inputTransparent = ColorUtil.getAlpha(color) < 255 && transparencyEnabled;
byte lastSkyLight = skyLightAtInputPos[yMin];
int quadBottomY = yMin;
int quadTopY = -1;
byte adjSkyLight = RenderDataPointUtil.getLightSky(adjPoint);
byte lightToApply;
// walk up the sky lights and create a new face
// whenever the light changes to different valid value
for (int i = yMin; i < yMax; i++)
if (!adjTransparent)
{
byte skyLight = skyLightAtInputPos[i];
if (skyLight != lastSkyLight)
{
// the sky light changed, create the in-progress face
tryAddVerticalFaceWithSkyLightToBuilder(
builder, direction,
x, z, horizontalWidth,
color, irisBlockMaterialId, blockLight,
lastSkyLight, inputTransparent, quadTopY, quadBottomY
// Adjacent is opaque
boolean adjacentCoversThis =
!adjacentIsSameDetailLevel
&& RenderDataPointUtil.getYMax(adjPoint) >= caveCullingMaxY
&&
(
(x == 0 && direction == EDhDirection.WEST)
|| (z == 0 && direction == EDhDirection.NORTH)
|| (x == 256 && direction == EDhDirection.EAST)
|| (z == 256 && direction == EDhDirection.SOUTH)
);
lastSkyLight = skyLight;
quadBottomY = i;
}
quadTopY = (i + 1);
lightToApply = adjacentCoversThis ? adjSkyLight : SKYLIGHT_COVERED;
}
else
{
// Adjacent is transparent, use below light
lightToApply = RenderDataPointUtil.getLightSky(adjBelowPoint);
}
// add the in-progress face if present
if (quadTopY != -1)
// Apply light to the range [adjMinY, adjMaxY)
applyLightToRange(segments, newSegments, adjMinY, adjMaxY, lightToApply);
// Fill overhang area [adjMaxY, adjAboveMinY) with adjSkyLight
short adjAboveMinY = RenderDataPointUtil.getYMin(adjAbovePoint);
if (adjMaxY < adjAboveMinY)
{
tryAddVerticalFaceWithSkyLightToBuilder(
builder, direction,
x, z, horizontalWidth,
color, irisBlockMaterialId, blockLight,
lastSkyLight, inputTransparent, quadTopY, quadBottomY
);
applyLightToRange(segments, newSegments, adjMaxY, adjAboveMinY, adjSkyLight);
}
}
finally
//=======================//
// Create vertical faces //
// from segments //
//=======================//
for (int i = 0; i < segments.size(); i++)
{
// clean up the array before the next thread uses it
// (may be unnecessary since we only work between the yMin-yMax anyway, but is helpful for debugging)
Arrays.fill(skyLightAtInputPos, yMin, yMax, SKYLIGHT_EMPTY);
long segment = segments.getLong(i);
tryAddVerticalFaceWithSkyLightToBuilder(
builder, direction,
x, z, horizontalWidth,
color, irisBlockMaterialId, blockLight,
YSegmentUtil.getSkyLight(segment), inputTransparent, YSegmentUtil.getEndY(segment), YSegmentUtil.getStartY(segment)
);
}
}
/**
* Apply the new light value over the given y range,
* splitting segments as needed
* <p>
* source: claude.ai
*/
private static void applyLightToRange(
LongArrayList segments, LongArrayList newSegments,
short rangeStart, short rangeEnd,
byte newLight)
{
// clear the pooled array that the new segments will go into
newSegments.clear();
for (int i = 0; i < segments.size(); i++)
{
long seg = segments.getLong(i);
short endY = YSegmentUtil.getEndY(seg);
short startY = YSegmentUtil.getStartY(seg);
byte skyLight = YSegmentUtil.getSkyLight(seg);
// No overlap
if (endY <= rangeStart
|| startY >= rangeEnd)
{
newSegments.add(seg);
continue;
}
// Partial or complete overlap - need to split
// Part before the range
if (startY < rangeStart)
{
newSegments.add(YSegmentUtil.encode(startY, rangeStart, skyLight));
}
// Overlapping part - take minimum light
short overlapStart = (short)Math.max(startY, rangeStart);
short overlapEnd = (short)Math.min(endY, rangeEnd);
byte minLight = (byte) Math.min(newLight, skyLight);
newSegments.add(YSegmentUtil.encode(overlapStart, overlapEnd, minLight));
// Part after the range
if (endY > rangeEnd)
{
newSegments.add(YSegmentUtil.encode(rangeEnd, endY, skyLight));
}
}
segments.clear();
segments.addAll(newSegments);
}
private static void tryAddVerticalFaceWithSkyLightToBuilder(
LodQuadBuilder builder, EDhDirection direction,
short x, short z, short horizontalWidth,
@@ -412,22 +433,72 @@ public class ColumnBox
)
{
// invalid positions will have a negative skylight
if (lastSkyLight >= 0)
if (lastSkyLight < 0)
{
// Don't add transparent vertical faces
// unless the adjacent position is empty.
// This is done to prevent walls between water blocks in the ocean.
if (!inputTransparent
|| (lastSkyLight == LodUtil.MAX_MC_LIGHT))
{
// don't add negative/empty height faces
short height = (short) (quadTopY - quadBottomY);
if (height > 0)
{
builder.addQuadAdj(direction, x, (short) quadBottomY, z, horizontalWidth, height, color, irisBlockMaterialId, lastSkyLight, blockLight);
}
}
return;
}
// Don't add transparent vertical faces
// unless the adjacent position is empty.
// This is done to prevent walls between water blocks in the ocean.
if (inputTransparent
&& (lastSkyLight != LodUtil.MAX_MC_LIGHT))
{
return;
}
// don't add negative/empty height faces
short height = (short) (quadTopY - quadBottomY);
if (height <= 0)
{
return;
}
builder.addQuadAdj(
direction,
x, (short) quadBottomY, z,
horizontalWidth, height,
color, irisBlockMaterialId, lastSkyLight, blockLight);
}
//================//
// helper classes //
//================//
/**
* encodes height/light data into a long
* to reduce object allocations.
*/
private static class YSegmentUtil
{
private static final int HEIGHT_WIDTH = Short.SIZE;
private static final int SKY_LIGHT_WIDTH = Byte.SIZE;
private static final int START_Y_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1;
private static final int END_Y_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1;
private static final int SKY_LIGHT_MASK = (int) Math.pow(2, SKY_LIGHT_WIDTH) - 1;
private static final int START_Y_OFFSET = 0;
private static final int END_Y_OFFSET = START_Y_OFFSET + HEIGHT_WIDTH;
private static final int SKY_LIGHT_OFFSET = END_Y_OFFSET + HEIGHT_WIDTH;
public static long encode(short startY, short endY, byte skyLight)
{
long data = 0L;
data |= (long) (startY & START_Y_MASK) << START_Y_OFFSET;
data |= (long) (endY & END_Y_MASK) << END_Y_OFFSET;
data |= (long) (skyLight & SKY_LIGHT_MASK) << SKY_LIGHT_OFFSET;
return data;
}
public static short getStartY(long data) { return (short) ((data >> START_Y_OFFSET) & START_Y_MASK); }
public static short getEndY(long data) { return (short) ((data >> END_Y_OFFSET) & END_Y_MASK); }
public static byte getSkyLight(long data) { return (byte) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); }
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java
+183 -180
View File
@@ -27,15 +27,15 @@ import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.render.glObject.GLProxy;
import com.seibel.distanthorizons.core.util.ColorUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import java.util.concurrent.CompletableFuture;
@@ -48,6 +48,8 @@ public class ColumnRenderBufferBuilder
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Column Buffer Builder");
//==============//
@@ -63,7 +65,7 @@ public class ColumnRenderBufferBuilder
{
DhBlockPos minBlockPos = new DhBlockPos(DhSectionPos.getMinCornerBlockX(pos), clientLevel.getLevelWrapper().getMinHeight(), DhSectionPos.getMinCornerBlockZ(pos));
LodBufferContainer bufferContainer = new LodBufferContainer(pos, minBlockPos);
CompletableFuture<LodBufferContainer> uploadFuture = bufferContainer.makeAndUploadBuffersAsync(quadBuilder, GLProxy.getInstance().getGpuUploadMethod());
CompletableFuture<LodBufferContainer> uploadFuture = bufferContainer.makeAndUploadBuffersAsync(quadBuilder);
uploadFuture.whenComplete((uploadedBuffer, exception) ->
{
// clean up if not uploaded
@@ -105,208 +107,209 @@ public class ColumnRenderBufferBuilder
// build each column //
//===================//
byte thisDetailLevel = renderSource.getDataDetailLevel();
for (int relX = 0; relX < ColumnRenderSource.SECTION_SIZE; relX++)
// pooled arrays for ColumnBox use
try (PhantomArrayListCheckout phantomArrayCheckout = ARRAY_LIST_POOL.checkoutArrays(0, 0, 2))
{
for (int relZ = 0; relZ < ColumnRenderSource.SECTION_SIZE; relZ++)
byte thisDetailLevel = renderSource.getDataDetailLevel();
for (int relX = 0; relX < ColumnRenderSource.WIDTH; relX++)
{
// stop the builder if requested
UncheckedInterruptedException.throwIfInterrupted();
// ignore empty/null columns
ColumnArrayView columnRenderData = renderSource.getVerticalDataPointView(relX, relZ);
if (columnRenderData.size() == 0
|| !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0))
|| RenderDataPointUtil.isVoid(columnRenderData.get(0)))
for (int relZ = 0; relZ < ColumnRenderSource.WIDTH; relZ++)
{
continue;
}
//=============//
// debug limit //
//=============//
// can be used to limit the buffer building to a specific relative position.
// useful for debugging a single column
if (columnBuilderDebugEnabled)
{
int wantedX = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugXRow.get();
if (wantedX >= 0 && relX != wantedX)
// ignore empty/null columns
ColumnArrayView columnRenderData = renderSource.getVerticalDataPointView(relX, relZ);
if (columnRenderData.size() == 0
|| !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0))
|| RenderDataPointUtil.hasZeroHeight(columnRenderData.get(0)))
{
continue;
}
int wantedZ = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugZRow.get();
if (wantedZ >= 0 && relZ != wantedZ)
//=============//
// debug limit //
//=============//
// can be used to limit the buffer building to a specific relative position.
// useful for debugging a single column
if (columnBuilderDebugEnabled)
{
continue;
}
}
//==================================//
// get adjacent render data columns //
//==================================//
ColumnArrayView[] adjColumnViews = new ColumnArrayView[EDhDirection.ADJ_DIRECTIONS.length];
for (EDhDirection lodDirection : EDhDirection.ADJ_DIRECTIONS)
{
try
{
int xAdj = relX + lodDirection.getNormal().x;
int zAdj = relZ + lodDirection.getNormal().z;
boolean isCrossRenderSourceBoundary =
(xAdj < 0 || xAdj >= ColumnRenderSource.SECTION_SIZE) ||
(zAdj < 0 || zAdj >= ColumnRenderSource.SECTION_SIZE);
ColumnRenderSource adjRenderSource;
byte adjDetailLevel;
//=========================//
// get the adjacent render //
// source if present //
//=========================//
if (!isCrossRenderSourceBoundary)
int wantedX = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugXRow.get();
if (wantedX >= 0 && relX != wantedX)
{
// the adjacent position is inside this same render source
adjRenderSource = renderSource;
adjDetailLevel = thisDetailLevel;
continue;
}
else
{
// the adjacent position is outside this render source
// skip empty sections
adjRenderSource = adjRegions[lodDirection.ordinal() - 2];
if (adjRenderSource == null)
{
continue;
}
adjDetailLevel = adjRenderSource.getDataDetailLevel();
if (adjDetailLevel == thisDetailLevel)
{
// if the adjacent position is outside this render source,
// wrap the position around so it's inside the adjacent source
if (xAdj < 0)
{
xAdj += ColumnRenderSource.SECTION_SIZE;
}
if (xAdj >= ColumnRenderSource.SECTION_SIZE)
{
xAdj -= ColumnRenderSource.SECTION_SIZE;
}
if (zAdj < 0)
{
zAdj += ColumnRenderSource.SECTION_SIZE;
}
if (zAdj >= ColumnRenderSource.SECTION_SIZE)
{
zAdj -= ColumnRenderSource.SECTION_SIZE;
}
}
}
//========================//
// get the adjacent views //
//========================//
// the old logic handled additional cases, but they never appeared to fire,
// so just these two cases should be fine
boolean expectedDetailLevels = (adjDetailLevel == thisDetailLevel) || (adjDetailLevel > thisDetailLevel);
if (!expectedDetailLevels)
{
LodUtil.assertNotReach("Mismatch between adjacent detail level ["+adjDetailLevel+"] and this render source's detail level ["+thisDetailLevel+"]. Detail levels should be adj >= this.");
}
adjColumnViews[lodDirection.ordinal() - 2] = adjRenderSource.getVerticalDataPointView(xAdj, zAdj);
}
catch (RuntimeException e)
{
LOGGER.warn("Failed to get adj data for relative pos: [" + thisDetailLevel + ":" + relX + "," + relZ + "] at [" + lodDirection + "], Error: [" + e.getMessage() + "].", e);
}
} // for adjacent directions
//==========================//
// build this render column //
//==========================//
ColumnRenderSource.DebugSourceFlag debugSourceFlag = renderSource.debugGetFlag(relX, relZ);
// We render every vertical lod present in this position
// We only stop when we find a block that is void or non-existing block
for (int i = 0; i < columnRenderData.size(); i++)
{
// can be uncommented to limit which vertical LOD is generated
if (Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugEnable.get())
{
int wantedColumnIndex = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugColumnIndex.get();
if (wantedColumnIndex >= 0 && i != wantedColumnIndex)
int wantedZ = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugZRow.get();
if (wantedZ >= 0 && relZ != wantedZ)
{
continue;
}
}
long data = columnRenderData.get(i);
// If the data is not render-able (Void or non-existing) we stop since there is
// no data left in this position
if (RenderDataPointUtil.isVoid(data) || !RenderDataPointUtil.doesDataPointExist(data))
//==================================//
// get adjacent render data columns //
//==================================//
ColumnArrayView[] adjColumnViews = new ColumnArrayView[EDhDirection.CARDINAL_COMPASS.length];
for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS)
{
break;
try
{
int xAdj = relX + direction.normal.x;
int zAdj = relZ + direction.normal.z;
boolean isCrossRenderSourceBoundary =
(xAdj < 0 || xAdj >= ColumnRenderSource.WIDTH) ||
(zAdj < 0 || zAdj >= ColumnRenderSource.WIDTH);
ColumnRenderSource adjRenderSource;
byte adjDetailLevel;
//=========================//
// get the adjacent render //
// source if present //
//=========================//
if (!isCrossRenderSourceBoundary)
{
// the adjacent position is inside this same render source
adjRenderSource = renderSource;
adjDetailLevel = thisDetailLevel;
}
else
{
// the adjacent position is outside this render source
// skip empty sections
adjRenderSource = adjRegions[direction.compassIndex];
if (adjRenderSource == null)
{
continue;
}
adjDetailLevel = adjRenderSource.getDataDetailLevel();
if (adjDetailLevel == thisDetailLevel)
{
// if the adjacent position is outside this render source,
// wrap the position around so it's inside the adjacent source
if (xAdj < 0)
{
xAdj += ColumnRenderSource.WIDTH;
}
if (xAdj >= ColumnRenderSource.WIDTH)
{
xAdj -= ColumnRenderSource.WIDTH;
}
if (zAdj < 0)
{
zAdj += ColumnRenderSource.WIDTH;
}
if (zAdj >= ColumnRenderSource.WIDTH)
{
zAdj -= ColumnRenderSource.WIDTH;
}
}
}
//========================//
// get the adjacent views //
//========================//
// the old logic handled additional cases, but they never appeared to fire,
// so just these two cases should be fine
boolean expectedDetailLevels = (adjDetailLevel == thisDetailLevel) || (adjDetailLevel > thisDetailLevel);
if (!expectedDetailLevels)
{
LodUtil.assertNotReach("Mismatch between adjacent detail level ["+adjDetailLevel+"] and this render source's detail level ["+thisDetailLevel+"]. Detail levels should be adj >= this.");
}
adjColumnViews[direction.compassIndex] = adjRenderSource.getVerticalDataPointView(xAdj, zAdj);
}
catch (RuntimeException e)
{
LOGGER.warn("Failed to get adj data for relative pos: [" + thisDetailLevel + ":" + relX + "," + relZ + "] at [" + direction + "], Error: [" + e.getMessage() + "].", e);
}
} // for adjacent directions
//==========================//
// build this render column //
//==========================//
ColumnRenderSource.DebugSourceFlag debugSourceFlag = renderSource.debugGetFlag(relX, relZ);
for (int i = 0; i < columnRenderData.size(); i++)
{
// can be uncommented to limit which vertical LOD is generated
if (Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugEnable.get())
{
int wantedColumnIndex = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugColumnIndex.get();
if (wantedColumnIndex >= 0
&& i != wantedColumnIndex)
{
continue;
}
}
long data = columnRenderData.get(i);
// If the data is not render-able (Void or non-existing) we stop since there is
// no data left in this position
if (RenderDataPointUtil.hasZeroHeight(data)
|| !RenderDataPointUtil.doesDataPointExist(data))
{
break;
}
long topDataPoint = (i - 1) >= 0 ? columnRenderData.get(i - 1) : RenderDataPointUtil.EMPTY_DATA;
long bottomDataPoint = (i + 1) < columnRenderData.size() ? columnRenderData.get(i + 1) : RenderDataPointUtil.EMPTY_DATA;
addRenderDataPointToBuilder(
clientLevel, phantomArrayCheckout,
data, topDataPoint, bottomDataPoint,
adjColumnViews, isSameDetailLevel,
thisDetailLevel, relX, relZ,
quadBuilder, debugSourceFlag);
}
long topDataPoint = (i - 1) >= 0 ? columnRenderData.get(i - 1) : RenderDataPointUtil.EMPTY_DATA;
long bottomDataPoint = (i + 1) < columnRenderData.size() ? columnRenderData.get(i + 1) : RenderDataPointUtil.EMPTY_DATA;
addLodToBuffer(
clientLevel,
data, topDataPoint, bottomDataPoint,
adjColumnViews, isSameDetailLevel,
thisDetailLevel, relX, relZ,
quadBuilder, debugSourceFlag);
}
}// for z
}// for x
}// for z
}// for x
}// phantom checkout
quadBuilder.mergeQuads();
}
private static void addLodToBuffer(
IDhClientLevel clientLevel,
long data, long topData, long bottomData,
private static void addRenderDataPointToBuilder(
IDhClientLevel clientLevel, PhantomArrayListCheckout phantomArrayCheckout,
long renderData, long topRenderData, long bottomRenderData,
ColumnArrayView[] adjColumnViews, boolean[] isSameDetailLevel,
byte detailLevel, int renderSourceOffsetPosX, int renderSourceOffsetPosZ,
LodQuadBuilder quadBuilder, ColumnRenderSource.DebugSourceFlag debugSource)
{
long sectionPos = DhSectionPos.encode(detailLevel, renderSourceOffsetPosX, renderSourceOffsetPosZ);
short width = (short) BitShiftUtil.powerOfTwo(detailLevel);
short xMin = (short) DhSectionPos.getMinCornerBlockX(sectionPos);
short yMin = RenderDataPointUtil.getYMin(data);
short zMin = (short) DhSectionPos.getMinCornerBlockZ(sectionPos);
short ySize = (short) (RenderDataPointUtil.getYMax(data) - yMin);
short blockWidth = (short) DhSectionPos.getDetailLevelWidthInBlocks(detailLevel);
short blockMinX = (short) DhSectionPos.getMinCornerBlockX(sectionPos);
short blockMinY = RenderDataPointUtil.getYMin(renderData);
short blockMinZ = (short) DhSectionPos.getMinCornerBlockZ(sectionPos);
short blockMaxY = (short) (RenderDataPointUtil.getYMax(renderData) - blockMinY);
if (ySize == 0)
if (blockMaxY == 0)
{
return;
}
else if (ySize < 0)
else if (blockMaxY < 0)
{
throw new IllegalArgumentException("Negative y size for the data! Data: [" + RenderDataPointUtil.toString(data) + "].");
throw new IllegalArgumentException("Negative y size for the renderDataPoint! Data: [" + RenderDataPointUtil.toString(renderData) + "].");
}
byte blockMaterialId = RenderDataPointUtil.getBlockMaterialId(data);
byte blockMaterialId = RenderDataPointUtil.getBlockMaterialId(renderData);
@@ -321,11 +324,11 @@ public class ColumnRenderBufferBuilder
float brightnessMultiplier = Config.Client.Advanced.Graphics.Quality.brightnessMultiplier.get().floatValue();
if (saturationMultiplier == 1.0 && brightnessMultiplier == 1.0)
{
color = RenderDataPointUtil.getColor(data);
color = RenderDataPointUtil.getColor(renderData);
}
else
{
float[] ahsv = ColorUtil.argbToAhsv(RenderDataPointUtil.getColor(data));
float[] ahsv = ColorUtil.argbToAhsv(RenderDataPointUtil.getColor(renderData));
color = ColorUtil.ahsvToArgb(ahsv[0], ahsv[1], ahsv[2] * saturationMultiplier, ahsv[3] * brightnessMultiplier);
}
break;
@@ -415,14 +418,14 @@ public class ColumnRenderBufferBuilder
}
ColumnBox.addBoxQuadsToBuilder(
quadBuilder, clientLevel,
width, ySize, width,
xMin, yMin, zMin,
quadBuilder, phantomArrayCheckout, clientLevel,
blockWidth, blockMaxY,
blockMinX, blockMinY, blockMinZ,
color,
blockMaterialId,
RenderDataPointUtil.getLightSky(data),
fullBright ? 15 : RenderDataPointUtil.getLightBlock(data),
topData, bottomData, adjColumnViews, isSameDetailLevel);
RenderDataPointUtil.getLightSky(renderData),
fullBright ? LodUtil.MAX_MC_LIGHT : RenderDataPointUtil.getLightBlock(renderData),
topRenderData, bottomRenderData, adjColumnViews, isSameDetailLevel);
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodBufferContainer.java
+9 -5
View File
@@ -82,7 +82,7 @@ public class LodBufferContainer implements AutoCloseable
//==================//
/** Should be run on a DH thread. */
public synchronized CompletableFuture<LodBufferContainer> makeAndUploadBuffersAsync(LodQuadBuilder builder, EDhApiGpuUploadMethod gpuUploadMethod)
public synchronized CompletableFuture<LodBufferContainer> makeAndUploadBuffersAsync(LodQuadBuilder builder)
{
// separate variable to prevent race condition when checking null
CompletableFuture<LodBufferContainer> future = this.uploadFuture;
@@ -117,6 +117,8 @@ public class LodBufferContainer implements AutoCloseable
throw new InterruptedException();
}
EDhApiGpuUploadMethod gpuUploadMethod = GLProxy.getInstance().getGpuUploadMethod();
// upload on the render thread
uploadBuffersDirect(this.vbos, opaqueBuffers, gpuUploadMethod);
uploadBuffersDirect(this.vbosTransparent, transparentBuffers, gpuUploadMethod);
@@ -177,7 +179,9 @@ public class LodBufferContainer implements AutoCloseable
}
return newVbos;
}
private static void uploadBuffersDirect(GLVertexBuffer[] vbos, ArrayList<ByteBuffer> byteBuffers, EDhApiGpuUploadMethod method) throws InterruptedException
private static void uploadBuffersDirect(
GLVertexBuffer[] vbos, ArrayList<ByteBuffer> byteBuffers,
EDhApiGpuUploadMethod uploadMethod) throws InterruptedException
{
int vboIndex = 0;
for (int i = 0; i < byteBuffers.size(); i++)
@@ -191,7 +195,7 @@ public class LodBufferContainer implements AutoCloseable
// get or create the VBO
if (vbos[vboIndex] == null)
{
vbos[vboIndex] = new GLVertexBuffer(method.useBufferStorage);
vbos[vboIndex] = new GLVertexBuffer(uploadMethod.useBufferStorage);
}
GLVertexBuffer vbo = vbos[vboIndex];
@@ -202,13 +206,13 @@ public class LodBufferContainer implements AutoCloseable
try
{
vbo.bind();
vbo.uploadBuffer(buffer, size / LodUtil.LOD_VERTEX_FORMAT.getByteSize(), method, FULL_SIZED_BUFFER);
vbo.uploadBuffer(buffer, size / LodUtil.LOD_VERTEX_FORMAT.getByteSize(), uploadMethod, FULL_SIZED_BUFFER);
}
catch (Exception e)
{
vbos[vboIndex] = null;
vbo.close();
LOGGER.error("Failed to upload buffer: ", e);
LOGGER.error("Failed to upload buffer. Error: ["+e.getMessage()+"].", e);
}
vboIndex++;
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java
+19 -13
View File
@@ -138,7 +138,8 @@ public class LodQuadBuilder
//===========//
public void addQuadAdj(
EDhDirection dir, short x, short y, short z,
EDhDirection dir,
short x, short y, short z,
short widthEastWest, short widthNorthSouthOrUpDown,
int color, byte irisBlockMaterialId, byte skyLight, byte blockLight)
{
@@ -149,11 +150,11 @@ public class LodQuadBuilder
BufferQuad quad = new BufferQuad(x, y, z, widthEastWest, widthNorthSouthOrUpDown, color, irisBlockMaterialId, skyLight, blockLight, dir);
ArrayList<BufferQuad> quadList = (this.doTransparency && ColorUtil.getAlpha(color) < 255) ? this.transparentQuads[dir.ordinal()] : this.opaqueQuads[dir.ordinal()];
if (!quadList.isEmpty() &&
(
quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.EastWest)
|| quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.NorthSouthOrUpDown))
)
if (!quadList.isEmpty()
&& (
quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.EastWest)
|| quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.NorthSouthOrUpDown))
)
{
this.premergeCount++;
return;
@@ -165,18 +166,23 @@ public class LodQuadBuilder
// XZ
public void addQuadUp(short minX, short maxY, short minZ, short widthEastWest, short widthNorthSouthOrUpDown, int color, byte irisBlockMaterialId, byte skylight, byte blocklight) // TODO argument names are wrong
{
BufferQuad quad = new BufferQuad(minX, maxY, minZ, widthEastWest, widthNorthSouthOrUpDown, color, irisBlockMaterialId, skylight, blocklight, EDhDirection.UP);
boolean isTransparent = (this.doTransparency && ColorUtil.getAlpha(color) < 255);
ArrayList<BufferQuad> quadList = isTransparent ? this.transparentQuads[EDhDirection.UP.ordinal()] : this.opaqueQuads[EDhDirection.UP.ordinal()];
ArrayList<BufferQuad> quadList = isTransparent
? this.transparentQuads[EDhDirection.UP.ordinal()]
: this.opaqueQuads[EDhDirection.UP.ordinal()];
BufferQuad quad = new BufferQuad(minX, maxY, minZ, widthEastWest, widthNorthSouthOrUpDown, color, irisBlockMaterialId, skylight, blocklight, EDhDirection.UP);
quadList.add(quad);
}
public void addQuadDown(short x, short y, short z, short width, short wz, int color, byte irisBlockMaterialId, byte skylight, byte blocklight)
{
ArrayList<BufferQuad> quadArray = (this.doTransparency && ColorUtil.getAlpha(color) < 255)
? this.transparentQuads[EDhDirection.DOWN.ordinal()]
: this.opaqueQuads[EDhDirection.DOWN.ordinal()];
BufferQuad quad = new BufferQuad(x, y, z, width, wz, color, irisBlockMaterialId, skylight, blocklight, EDhDirection.DOWN);
ArrayList<BufferQuad> qs = (doTransparency && ColorUtil.getAlpha(color) < 255)
? transparentQuads[EDhDirection.DOWN.ordinal()] : opaqueQuads[EDhDirection.DOWN.ordinal()];
qs.add(quad);
quadArray.add(quad);
}
@@ -304,7 +310,7 @@ public class LodQuadBuilder
short widthEastWest = quad.widthEastWest;
short widthNorthSouth = quad.widthNorthSouthOrUpDown;
byte normalIndex = (byte) quad.direction.ordinal();
EDhDirection.Axis axis = quad.direction.getAxis();
EDhDirection.Axis axis = quad.direction.axis;
for (int i = 0; i < quadBase.length; i++)
{
short dx, dy, dz;
@@ -352,7 +358,7 @@ public class LodQuadBuilder
if (this.grassSideRenderingMode != EDhApiGrassSideRendering.AS_GRASS)
{
// only change the vertex color if it's on the side or bottom
if (quad.direction.getAxis().isHorizontal() || quad.direction == EDhDirection.DOWN)
if (quad.direction.axis.isHorizontal() || quad.direction == EDhDirection.DOWN)
{
if (this.grassSideRenderingMode == EDhApiGrassSideRendering.AS_DIRT
// if we want the color to fade, only apply the dirt color to the bottom vertices
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java
+1 -3
View File
@@ -101,9 +101,7 @@ public final class ColumnArrayView implements IColumnDataView
@Override
public ColumnArrayView subView(int dataIndexStart, int dataCount)
{
return new ColumnArrayView(data, dataCount * verticalSize, offset + dataIndexStart * verticalSize, verticalSize);
}
{ return new ColumnArrayView(data, dataCount * verticalSize, offset + dataIndexStart * verticalSize, verticalSize); }
public void fill(long value) { Arrays.fill(data.elements(), offset, offset + size, value); }
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataOcclusionCuller.java
+5 -5
View File
@@ -22,11 +22,11 @@ public class FullDataOcclusionCuller
int relX, int relZ
)
{
LongArrayList centerColumn = dataSource.get(relX, relZ);
LongArrayList posXColumn = dataSource.tryGet(relX + 1, relZ);
LongArrayList negXColumn = dataSource.tryGet(relX - 1, relZ);
LongArrayList posZColumn = dataSource.tryGet(relX, relZ + 1);
LongArrayList negZColumn = dataSource.tryGet(relX, relZ - 1);
LongArrayList centerColumn = dataSource.getColumnAtRelPos(relX, relZ);
LongArrayList posXColumn = dataSource.tryGetColumnAtRelPos(relX + 1, relZ);
LongArrayList negXColumn = dataSource.tryGetColumnAtRelPos(relX - 1, relZ);
LongArrayList posZColumn = dataSource.tryGetColumnAtRelPos(relX, relZ + 1);
LongArrayList negZColumn = dataSource.tryGetColumnAtRelPos(relX, relZ - 1);
if (posXColumn == null || posXColumn.size() == 0
|| negXColumn == null || negXColumn.size() == 0
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java
+22 -35
View File
@@ -31,6 +31,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPosMutable;
import com.seibel.distanthorizons.core.render.LodQuadTree;
import com.seibel.distanthorizons.core.util.*;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
@@ -66,7 +67,8 @@ public class FullDataToRenderDataTransformer
//==============================//
@Nullable
public static ColumnRenderSource transformFullDataToRenderSource(@Nullable FullDataSourceV2 fullDataSource, @Nullable IClientLevelWrapper levelWrapper)
public static ColumnRenderSource transformFullDataToRenderSource(
@Nullable FullDataSourceV2 fullDataSource, @Nullable IClientLevelWrapper levelWrapper)
{
if (fullDataSource == null)
{
@@ -102,7 +104,8 @@ public class FullDataToRenderDataTransformer
* @throws InterruptedException Can be caused by interrupting the thread upstream.
* Generally thrown if the method is running after the client leaves the current world.
*/
private static ColumnRenderSource transformCompleteFullDataToColumnData(IClientLevelWrapper levelWrapper, FullDataSourceV2 fullDataSource) throws InterruptedException
private static ColumnRenderSource transformCompleteFullDataToColumnData(
IClientLevelWrapper levelWrapper, FullDataSourceV2 fullDataSource) throws InterruptedException
{
final long pos = fullDataSource.getPos();
final byte dataDetail = fullDataSource.getDataDetailLevel();
@@ -126,7 +129,7 @@ public class FullDataToRenderDataTransformer
for (int z = 0; z < FullDataSourceV2.WIDTH; z++)
{
ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z);
LongArrayList dataColumn = fullDataSource.get(x, z);
LongArrayList dataColumn = fullDataSource.getColumnAtRelPos(x, z);
updateOrReplaceRenderDataViewColumnWithFullDataColumn(
levelWrapper, fullDataSource,
@@ -136,7 +139,7 @@ public class FullDataToRenderDataTransformer
}
}
columnSource.fillDebugFlag(0, 0, ColumnRenderSource.SECTION_SIZE, ColumnRenderSource.SECTION_SIZE, ColumnRenderSource.DebugSourceFlag.FULL);
columnSource.fillDebugFlag(0, 0, ColumnRenderSource.WIDTH, ColumnRenderSource.WIDTH, ColumnRenderSource.DebugSourceFlag.FULL);
return columnSource;
}
@@ -171,6 +174,7 @@ public class FullDataToRenderDataTransformer
// expand the ColumnArrayView to fit the new larger max vertical size
ColumnArrayView newColumnArrayView = new ColumnArrayView(dataArrayList, fullDataLength, 0, fullDataLength);
setRenderColumnView(levelWrapper, fullDataSource, blockX, blockZ, newColumnArrayView, fullDataColumn);
columnArrayView.changeVerticalSizeFrom(newColumnArrayView);
}
finally
@@ -275,18 +279,18 @@ public class FullDataToRenderDataTransformer
if (caveBlock)
{
if (caveCullingEnabled
// assume this data point is underground if it has no sky-light
&& skyLight == LodUtil.MIN_MC_LIGHT
// ignore caves above a certain height to prevent floating islands from having walls underneath them
&& topY < caveCullingMaxY
// cave culling shouldn't happen when at the top of the world
&& renderDataIndex != 0 && fullDataIndex != 0
// cave culling can't happen when at the bottom of the world
&& (fullDataIndex+1) < fullColumnData.size())
// assume this data point is underground if it has no sky-light
&& skyLight == LodUtil.MIN_MC_LIGHT
// ignore caves above a certain height to prevent floating islands from having walls underneath them
&& topY < caveCullingMaxY
// cave culling shouldn't happen when at the top of the world
&& renderDataIndex != 0 && fullDataIndex != 0
// cave culling can't happen when at the bottom of the world
&& (fullDataIndex + 1) < fullColumnData.size())
{
// we need to get the next sky/block lights because
// the air block here will always have a light of 0/0 due to only the top of the LOD's light being saved.
long nextFullData = fullColumnData.getLong(fullDataIndex+1);
long nextFullData = fullColumnData.getLong(fullDataIndex + 1);
int nextSkyLight = FullDataPointUtil.getSkyLight(nextFullData);
if (nextSkyLight == LodUtil.MIN_MC_LIGHT
@@ -320,10 +324,10 @@ public class FullDataToRenderDataTransformer
// non-solid block check //
//=======================//
if (ignoreNonCollidingBlocks
&& !block.isSolid()
&& !block.isLiquid()
&& block.getOpacity() != LodUtil.BLOCK_FULLY_OPAQUE)
if (ignoreNonCollidingBlocks
&& !block.isSolid()
&& !block.isLiquid()
&& block.getOpacity() != LodUtil.BLOCK_FULLY_OPAQUE)
{
if (colorBelowWithAvoidedBlocks)
{
@@ -333,7 +337,7 @@ public class FullDataToRenderDataTransformer
// this prevents issues if grass is transparent
if (ColorUtil.getAlpha(tempColor) != 0)
{
colorToApplyToNextBlock = ColorUtil.setAlpha(tempColor,255);
colorToApplyToNextBlock = ColorUtil.setAlpha(tempColor, 255);
skylightToApplyToNextBlock = skyLight;
blocklightToApplyToNextBlock = blockLight;
}
@@ -396,21 +400,4 @@ public class FullDataToRenderDataTransformer
//================//
// helper methods //
//================//
/**
* Called in loops that may run for an extended period of time. <br>
* This is necessary to allow canceling these transformers since running
* them after the client has left a given world will throw exceptions.
*/
private static void throwIfThreadInterrupted() throws InterruptedException
{
if (Thread.interrupted())
{
throw new InterruptedException(FullDataToRenderDataTransformer.class.getSimpleName() + " task interrupted.");
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java
+1 -4
View File
@@ -24,8 +24,6 @@ import java.util.List;
import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.api.interfaces.block.IDhApiBiomeWrapper;
import com.seibel.distanthorizons.api.interfaces.block.IDhApiBlockStateWrapper;
import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiChunkProcessingEvent;
import com.seibel.distanthorizons.api.objects.data.DhApiChunk;
import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint;
@@ -47,7 +45,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import com.seibel.distanthorizons.core.logging.DhLogger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class LodDataBuilder
@@ -142,7 +139,7 @@ public class LodDataBuilder
int columnZ = relBlockZ + chunkOffsetZ;
// Get column data
LongArrayList longs = dataSource.get(columnX, columnZ);
LongArrayList longs = dataSource.getColumnAtRelPos(columnX, columnZ);
if (longs == null)
{
longs = new LongArrayList(dataCapacity);
core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java
+104 -492
View File
@@ -19,496 +19,165 @@
package com.seibel.distanthorizons.core.enums;
import java.util.Arrays;
import java.util.Locale;
import java.util.Map;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import com.seibel.distanthorizons.core.util.math.Vec3i;
/**
* An (almost) exact copy of Minecraft's
* Direction enum. <Br><Br>
*
* Up <Br>
* Down <Br>
* North <Br>
* South <Br>
* East <Br>
* West <Br>
*
* @author James Seibel
* @version 2021-11-13
*/
public enum EDhDirection
{
/** negative Y */
DOWN(0, 1, -1, "down", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Y, new Vec3i(0, -1, 0)),
DOWN("down", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Y, new Vec3i(0, -1, 0), -1),
/** positive Y */
UP(1, 0, -1, "up", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Y, new Vec3i(0, 1, 0)),
UP("up", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Y, new Vec3i(0, 1, 0), -1),
/** negative Z */
NORTH(2, 3, 2, "north", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, -1)),
NORTH("north", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, -1), 0),
/** positive Z */
SOUTH(3, 2, 0, "south", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, 1)),
SOUTH("south", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, 1), 1),
/** negative X */
WEST(4, 5, 1, "west", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.X, new Vec3i(-1, 0, 0)),
WEST("west", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.X, new Vec3i(-1, 0, 0), 2),
/** positive X */
EAST(5, 4, 3, "east", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.X, new Vec3i(1, 0, 0));
EAST("east", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.X, new Vec3i(1, 0, 0), 3);
/**
* Up, Down, West, East, North, South <br>
* Similar to {@link EDhDirection#OPPOSITE_DIRECTIONS}, just with a different order
*/
public static final EDhDirection[] CARDINAL_DIRECTIONS = new EDhDirection[]{
/** Up, Down, West, East, North, South */
public static final EDhDirection[] ALL = new EDhDirection[] {
EDhDirection.UP,
EDhDirection.DOWN,
EDhDirection.WEST,
EDhDirection.EAST,
EDhDirection.NORTH,
EDhDirection.SOUTH};
EDhDirection.SOUTH
};
/**
* Up, Down, South, North, East, West <br>
* Similar to {@link EDhDirection#CARDINAL_DIRECTIONS}, just with a different order
*/
public static final EDhDirection[] OPPOSITE_DIRECTIONS = new EDhDirection[]{
EDhDirection.UP,
EDhDirection.DOWN,
EDhDirection.SOUTH,
EDhDirection.NORTH,
EDhDirection.EAST,
EDhDirection.WEST};
/** North, South, East, West */ // TODO rename to state this is just X/Z or flat directions
public static final EDhDirection[] ADJ_DIRECTIONS = new EDhDirection[]{
/** North, South, East, West */
public static final EDhDirection[] CARDINAL_COMPASS = new EDhDirection[] {
EDhDirection.EAST,
EDhDirection.WEST,
EDhDirection.SOUTH,
EDhDirection.NORTH};
// private final int data3d;
// private final int oppositeIndex;
// private final int data2d;
EDhDirection.NORTH
};
private final String name;
private final EDhDirection.Axis axis;
private final EDhDirection.AxisDirection axisDirection;
private final Vec3i normal;
private static final EDhDirection[] VALUES = values();
private static final Map<String, EDhDirection> BY_NAME = Arrays.stream(VALUES).collect(Collectors.toMap(EDhDirection::getName, (p_199787_0_) ->
public final String name;
public final EDhDirection.Axis axis;
public final EDhDirection.AxisDirection axisDirection;
public final Vec3i normal;
/** -1 if not a {@link EDhDirection#CARDINAL_COMPASS} direction */
public final int compassIndex;
//=============//
// constructor //
//=============//
EDhDirection(String name, EDhDirection.AxisDirection axisDirection, EDhDirection.Axis axis, Vec3i normal, int compassIndex)
{
return p_199787_0_;
}));
// private static final LodDirection[] BY_3D_DATA = Arrays.stream(VALUES).sorted(Comparator.comparingInt((p_199790_0_) ->
// {
// return p_199790_0_.data3d;
// })).toArray((p_199788_0_) ->
// {
// return new LodDirection[p_199788_0_];
// });
//
// private static final LodDirection[] BY_2D_DATA = Arrays.stream(VALUES).filter((p_199786_0_) ->
// {
// return p_199786_0_.getAxis().isHorizontal();
// }).sorted(Comparator.comparingInt((p_199789_0_) ->
// {
// return p_199789_0_.data2d;
// })).toArray((p_199791_0_) ->
// {
// return new LodDirection[p_199791_0_];
// });
// private static final Long2ObjectMap<LodDirection> BY_NORMAL = Arrays.stream(VALUES).collect(Collectors.toMap((p_218385_0_) ->
// {
// return (new BlockPos(p_218385_0_.getNormal())).asLong();
// }, (p_218384_0_) ->
// {
// return p_218384_0_;
// }, (p_218386_0_, p_218386_1_) ->
// {
// throw new IllegalArgumentException("Duplicate keys");
// }, Long2ObjectOpenHashMap::new));
EDhDirection(int p_i46016_3_, int p_i46016_4_, int p_i46016_5_, String p_i46016_6_, EDhDirection.AxisDirection p_i46016_7_, EDhDirection.Axis p_i46016_8_, Vec3i p_i46016_9_)
{
// this.data3d = p_i46016_3_;
// this.data2d = p_i46016_5_;
// this.oppositeIndex = p_i46016_4_;
this.name = p_i46016_6_;
this.axis = p_i46016_8_;
this.axisDirection = p_i46016_7_;
this.normal = p_i46016_9_;
this.name = name;
this.axis = axis;
this.axisDirection = axisDirection;
this.normal = normal;
this.compassIndex = compassIndex;
}
// public static LodDirection[] orderedByNearest(Entity p_196054_0_)
// {
// float f = p_196054_0_.getViewXRot(1.0F) * ((float) Math.PI / 180F);
// float f1 = -p_196054_0_.getViewYRot(1.0F) * ((float) Math.PI / 180F);
// float f2 = MathHelper.sin(f);
// float f3 = MathHelper.cos(f);
// float f4 = MathHelper.sin(f1);
// float f5 = MathHelper.cos(f1);
// boolean flag = f4 > 0.0F;
// boolean flag1 = f2 < 0.0F;
// boolean flag2 = f5 > 0.0F;
// float f6 = flag ? f4 : -f4;
// float f7 = flag1 ? -f2 : f2;
// float f8 = flag2 ? f5 : -f5;
// float f9 = f6 * f3;
// float f10 = f8 * f3;
// LodDirection lodDirection = flag ? EAST : WEST;
// LodDirection direction1 = flag1 ? UP : DOWN;
// LodDirection direction2 = flag2 ? SOUTH : NORTH;
// if (f6 > f8)
// {
// if (f7 > f9)
// {
// return makeDirectionArray(direction1, lodDirection, direction2);
// }
// else
// {
// return f10 > f7 ? makeDirectionArray(lodDirection, direction2, direction1) : makeDirectionArray(lodDirection, direction1, direction2);
// }
// }
// else if (f7 > f10)
// {
// return makeDirectionArray(direction1, direction2, lodDirection);
// }
// else
// {
// return f9 > f7 ? makeDirectionArray(direction2, lodDirection, direction1) : makeDirectionArray(direction2, direction1, lodDirection);
// }
// }
// private static LodDirection[] makeDirectionArray(LodDirection p_196053_0_, LodDirection p_196053_1_, LodDirection p_196053_2_)
// {
// return new LodDirection[] { p_196053_0_, p_196053_1_, p_196053_2_, p_196053_2_.getOpposite(), p_196053_1_.getOpposite(), p_196053_0_.getOpposite() };
// }
// public static LodDirection rotate(Mat4f p_229385_0_, LodDirection p_229385_1_)
// {
// Vec3i Vec3i = p_229385_1_.getNormal();
// Vector4f vector4f = new Vector4f(Vec3i.getX(), Vec3i.getY(), Vec3i.getZ(), 0.0F);
// vector4f.transform(p_229385_0_);
// return getNearest(vector4f.x(), vector4f.y(), vector4f.z());
// }
// public Quaternion getRotation()
// {
// Quaternion quaternion = Vector3f.XP.rotationDegrees(90.0F);
// switch (this)
// {
// case DOWN:
// return Vector3f.XP.rotationDegrees(180.0F);
// case UP:
// return Quaternion.ONE.copy();
// case NORTH:
// quaternion.mul(Vector3f.ZP.rotationDegrees(180.0F));
// return quaternion;
// case SOUTH:
// return quaternion;
// case WEST:
// quaternion.mul(Vector3f.ZP.rotationDegrees(90.0F));
// return quaternion;
// case EAST:
// default:
// quaternion.mul(Vector3f.ZP.rotationDegrees(-90.0F));
// return quaternion;
// }
// }
// public int get3DDataValue()
// {
// return this.data3d;
// }
//
// public int get2DDataValue()
// {
// return this.data2d;
// }
public EDhDirection.AxisDirection getAxisDirection()
{
return this.axisDirection;
}
// public LodDirection getOpposite()
// {
// return from3DDataValue(this.oppositeIndex);
// }
public EDhDirection getClockWise()
{
switch (this)
//=========//
// methods //
//=========//
public EDhDirection opposite()
{
switch(this)
{
case UP:
return EDhDirection.DOWN;
case DOWN:
return EDhDirection.UP;
case NORTH:
return EAST;
return EDhDirection.SOUTH;
case SOUTH:
return WEST;
case WEST:
return NORTH;
return EDhDirection.NORTH;
case EAST:
return SOUTH;
default:
throw new IllegalStateException("Unable to get Y-rotated facing of " + this);
}
}
public EDhDirection getCounterClockWise()
{
switch (this)
{
case NORTH:
return WEST;
case SOUTH:
return EAST;
return EDhDirection.WEST;
case WEST:
return SOUTH;
case EAST:
return NORTH;
return EDhDirection.EAST;
default:
throw new IllegalStateException("Unable to get CCW facing of " + this);
throw new IllegalArgumentException();
}
}
public String getName()
{
return this.name;
}
public EDhDirection.Axis getAxis()
{
return this.axis;
}
@Override
public String toString() { return this.name; }
public static EDhDirection byName(String name)
{
return name == null ? null : BY_NAME.get(name.toLowerCase(Locale.ROOT));
}
// public static LodDirection from3DDataValue(int p_82600_0_)
// {
// return BY_3D_DATA[MathHelper.abs(p_82600_0_ % BY_3D_DATA.length)];
// }
//
// public static LodDirection from2DDataValue(int p_176731_0_)
// {
// return BY_2D_DATA[MathHelper.abs(p_176731_0_ % BY_2D_DATA.length)];
// }
// @Nullable
// public static LodDirection fromNormal(int p_218383_0_, int p_218383_1_, int p_218383_2_)
// {
// return BY_NORMAL.get(BlockPos.asLong(p_218383_0_, p_218383_1_, p_218383_2_));
// }
// public static LodDirection fromYRot(double p_176733_0_)
// {
// return from2DDataValue(MathHelper.floor(p_176733_0_ / 90.0D + 0.5D) & 3);
// }
public static EDhDirection fromAxisAndDirection(EDhDirection.Axis p_211699_0_, EDhDirection.AxisDirection p_211699_1_)
{
switch (p_211699_0_)
{
case X:
return p_211699_1_ == EDhDirection.AxisDirection.POSITIVE ? EAST : WEST;
case Y:
return p_211699_1_ == EDhDirection.AxisDirection.POSITIVE ? UP : DOWN;
case Z:
default:
return p_211699_1_ == EDhDirection.AxisDirection.POSITIVE ? SOUTH : NORTH;
}
}
// public float toYRot()
// {
// return (this.data2d & 3) * 90;
// }
// public static LodDirection getRandom(Random p_239631_0_)
// {
// return Util.getRandom(VALUES, p_239631_0_);
// }
// public static LodDirection getNearest(double p_210769_0_, double p_210769_2_, double p_210769_4_)
// {
// return getNearest((float) p_210769_0_, (float) p_210769_2_, (float) p_210769_4_);
// }
// public static LodDirection getNearest(float p_176737_0_, float p_176737_1_, float p_176737_2_)
// {
// LodDirection lodDirection = NORTH;
// float f = Float.MIN_VALUE;
//
// for (LodDirection direction1 : VALUES)
// {
// float f1 = p_176737_0_ * direction1.normal.x + p_176737_1_ * direction1.normal.y + p_176737_2_ * direction1.normal.z;
// if (f1 > f)
// {
// f = f1;
// lodDirection = direction1;
// }
// }
//
// return lodDirection;
// }
public static EDhDirection get(EDhDirection.AxisDirection p_181076_0_, EDhDirection.Axis p_181076_1_)
{
for (EDhDirection lodDirection : VALUES)
{
if (lodDirection.getAxisDirection() == p_181076_0_ && lodDirection.getAxis() == p_181076_1_)
{
return lodDirection;
}
}
throw new IllegalArgumentException("No such direction: " + p_181076_0_ + " " + p_181076_1_);
}
//================//
// helper classes //
//================//
public Vec3i getNormal()
/**
* X <br>
* Y <br>
* Z <br>
*/
public enum Axis
{
return this.normal;
}
// public boolean isFacingAngle(float p_243532_1_)
// {
// float f = p_243532_1_ * ((float) Math.PI / 180F);
// float f1 = -MathHelper.sin(f);
// float f2 = MathHelper.cos(f);
// return this.normal.getX() * f1 + this.normal.getZ() * f2 > 0.0F;
// }
public enum Axis implements Predicate<EDhDirection>
{
X("x")
{
@Override
public int choose(int x, int y, int z)
{
return x;
}
@Override
public double choose(double x, double y, double z)
{
return x;
}
},
Y("y")
{
@Override
public int choose(int x, int y, int z)
{
return y;
}
@Override
public double choose(double x, double y, double z)
{
return y;
}
},
Z("z")
{
@Override
public int choose(int x, int y, int z)
{
return z;
}
@Override
public double choose(double x, double y, double z)
{
return z;
}
};
X("x"),
Y("y"),
Z("z");
private static final EDhDirection.Axis[] VALUES = values();
public final String name;
private static final Map<String, EDhDirection.Axis> BY_NAME = Arrays.stream(VALUES).collect(Collectors.toMap(EDhDirection.Axis::getName, (p_199785_0_) ->
{
return p_199785_0_;
}));
private final String name;
Axis(String name)
{
this.name = name;
}
//=============//
// constructor //
//=============//
public static EDhDirection.Axis byName(String name)
{
return BY_NAME.get(name.toLowerCase(Locale.ROOT));
}
Axis(String name) { this.name = name; }
public String getName()
{
return this.name;
}
public boolean isVertical()
{
return this == Y;
}
public boolean isHorizontal()
{
return this == X || this == Z;
}
//=========//
// methods //
//=========//
public boolean isVertical() { return this == Y; }
public boolean isHorizontal() { return this == X || this == Z; }
@Override
public String toString()
{
return this.name;
}
// public static LodDirection.Axis getRandom(Random p_239634_0_)
// {
// return Util.getRandom(VALUES, p_239634_0_);
// }
public String toString() { return this.name; }
@Override
public boolean test(EDhDirection p_test_1_)
{
return p_test_1_ != null && p_test_1_.getAxis() == this;
}
// public LodDirection.Plane getPlane()
// {
// switch (this)
// {
// case X:
// case Z:
// return LodDirection.Plane.HORIZONTAL;
// case Y:
// return LodDirection.Plane.VERTICAL;
// default:
// throw new Error("Someone's been tampering with the universe!");
// }
// }
public abstract int choose(int p_196052_1_, int p_196052_2_, int p_196052_3_);
public abstract double choose(double p_196051_1_, double p_196051_3_, double p_196051_5_);
}
/**
* POSITIVE <br>
* NEGATIVE <br>
*/
public enum AxisDirection
{
POSITIVE(1, "Towards positive"),
NEGATIVE(-1, "Towards negative");
private final int step;
private final String name;
public final int step;
public final String name;
//=============//
// constructor //
//=============//
AxisDirection(int newStep, String newName)
{
@@ -516,77 +185,20 @@ public enum EDhDirection
this.name = newName;
}
public int getStep()
{
return this.step;
}
//=========//
// methods //
//=========//
public EDhDirection.AxisDirection opposite()
{ return (this == POSITIVE) ? NEGATIVE : POSITIVE; }
@Override
public String toString()
{
return this.name;
}
public String toString() { return this.name; }
public EDhDirection.AxisDirection opposite()
{
return this == POSITIVE ? NEGATIVE : POSITIVE;
}
}
// public static enum Plane implements Iterable<LodDirection>, Predicate<LodDirection>
// {
// HORIZONTAL(new LodDirection[] { LodDirection.NORTH, LodDirection.EAST, LodDirection.SOUTH, LodDirection.WEST }, new LodDirection.Axis[] { LodDirection.Axis.X, LodDirection.Axis.Z }),
// VERTICAL(new LodDirection[] { LodDirection.UP, LodDirection.DOWN }, new LodDirection.Axis[] { LodDirection.Axis.Y });
//
// private final LodDirection[] faces;
// private final LodDirection.Axis[] axis;
//
// private Plane(LodDirection[] p_i49393_3_, LodDirection.Axis[] p_i49393_4_)
// {
// this.faces = p_i49393_3_;
// this.axis = p_i49393_4_;
// }
//
// public LodDirection getRandomDirection(Random p_179518_1_)
// {
// return Util.getRandom(this.faces, p_179518_1_);
// }
//
// public LodDirection.Axis getRandomAxis(Random p_244803_1_)
// {
// return Util.getRandom(this.axis, p_244803_1_);
// }
//
// @Override
// public boolean test(@Nullable LodDirection p_test_1_)
// {
// return p_test_1_ != null && p_test_1_.getAxis().getPlane() == this;
// }
//
// @Override
// public Iterator<LodDirection> iterator()
// {
// return Iterators.forArray(this.faces);
// }
//
// public Stream<LodDirection> stream()
// {
// return Arrays.stream(this.faces);
// }
// }
public String getSerializedName()
{
return this.name;
}
@Override
public String toString()
{
return this.name;
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java
-376
View File
@@ -1,376 +0,0 @@
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import com.seibel.distanthorizons.core.sql.dto.IBaseDTO;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.logging.DhLogger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
/**
* @see FullDataSourceProviderV2
* @see RemoteFullDataSourceProvider
* @see GeneratedFullDataSourceProvider
*/
public abstract class AbstractDataSourceHandler
<TDataSource extends IDataSource<TDhLevel>,
TDTO extends IBaseDTO<Long>,
TRepo extends AbstractDhRepo<Long, TDTO>,
TDhLevel extends IDhLevel>
implements AutoCloseable
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
private static final Set<String> CORRUPT_DATA_ERRORS_LOGGED = Collections.newSetFromMap(new ConcurrentHashMap<>());
/**
* The highest numerical detail level possible.
* Used when determining which positions to update.
*
* @see AbstractDataSourceHandler#MIN_SECTION_DETAIL_LEVEL
*/
public static final byte TOP_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + LodUtil.REGION_DETAIL_LEVEL;
/**
* The lowest numerical detail level possible.
*
* @see AbstractDataSourceHandler#TOP_SECTION_DETAIL_LEVEL
*/
public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
protected final PositionalLockProvider updateLockProvider = new PositionalLockProvider();
/**
* generally just used for debugging,
* keeps track of which positions are currently locked.
*/
public final Set<Long> lockedPosSet = ConcurrentHashMap.newKeySet();
public final ConcurrentHashMap<Long, AtomicInteger> queuedUpdateCountsByPos = new ConcurrentHashMap<>();
protected final ReentrantLock closeLock = new ReentrantLock();
protected volatile boolean isShutdown = false;
protected final TDhLevel level;
protected final File saveDir;
public final TRepo repo;
public final ArrayList<IDataSourceUpdateFunc<TDataSource>> dateSourceUpdateListeners = new ArrayList<>();
//=============//
// constructor //
//=============//
public AbstractDataSourceHandler(TDhLevel level, ISaveStructure saveStructure) { this(level, saveStructure, null); }
public AbstractDataSourceHandler(TDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride)
{
this.level = level;
this.saveDir = (saveDirOverride == null) ? saveStructure.getSaveFolder(level.getLevelWrapper()) : saveDirOverride;
this.repo = this.createRepo();
}
//==================//
// abstract methods //
//==================//
/** When this is called the parent folders should be created */
protected abstract TRepo createRepo();
protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException, DataCorruptedException;
protected abstract TDTO createDtoFromDataSource(TDataSource dataSource);
protected abstract TDataSource makeEmptyDataSource(long pos);
//==============//
// data reading //
//==============//
/**
* Returns the {@link TDataSource} for the given section position. <Br>
* The returned data source may be null if repo is in the process of shutting down. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
public CompletableFuture<TDataSource> getAsync(long pos)
{
AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
try
{
return CompletableFuture.supplyAsync(() -> this.get(pos), executor);
}
catch (RejectedExecutionException ignore)
{
// the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back
return CompletableFuture.completedFuture(null);
}
}
/**
* Should only be used in internal file handler methods where we are already running on a file handler thread.
* Can return null if the repo is in the process of being shut down
* @see AbstractDataSourceHandler#getAsync(long)
*/
@Nullable
public TDataSource get(long pos)
{
TDataSource dataSource = null;
try(TDTO dto = this.repo.getByKey(pos))
{
if (dto != null)
{
try
{
// load from database
dataSource = this.createDataSourceFromDto(dto);
}
catch (DataCorruptedException e)
{
// there's a rare issue where the exception doesn't
// have a message, which can cause problems
String message = (e.getMessage() == null) ? e.getMessage() : "No Error message for exception ["+e.getClass().getSimpleName()+"]";
// Only log each message type once.
// This is done to prevent logging "No compression mode with the value [2]" 10,000 times
// if the user is migrating from a nightly build and used ZStd.
if (CORRUPT_DATA_ERRORS_LOGGED.add(message))
{
LOGGER.warn("Corrupted data found at pos [" + DhSectionPos.toString(pos) + "]. Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + message + "].", e);
}
this.repo.deleteWithKey(pos);
}
}
else
{
dataSource = this.makeEmptyDataSource(pos);
}
}
catch (InterruptedException ignore) { }
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+ DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e);
}
return dataSource;
}
//===============//
// data updating //
//===============//
/**
* Can be used if the same thread is already handling IO and/or LOD generation.
* Otherwise the async version {@link AbstractDataSourceHandler#updateDataSourceAsync(FullDataSourceV2)} may be a better choice.
*/
public void updateDataSource(@NotNull FullDataSourceV2 inputDataSource)
{ this.updateDataSourceAtPos(inputDataSource.getPos(), inputDataSource, true); }
/**
* Can be used if you don't want to lock the current thread
* Otherwise the sync version {@link AbstractDataSourceHandler#updateDataSource(FullDataSourceV2)} may be a better choice.
*/
public CompletableFuture<Void> updateDataSourceAsync(@NotNull FullDataSourceV2 inputDataSource)
{
AbstractExecutorService executor = ThreadPoolUtil.getChunkToLodBuilderExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
try
{
this.markUpdateStart(inputDataSource.getPos());
return CompletableFuture.runAsync(() ->
{
try
{
this.updateDataSourceAtPos(inputDataSource.getPos(), inputDataSource, true);
}
catch (Exception e)
{
LOGGER.error("Unexpected error in async data source update at pos: ["+DhSectionPos.toString(inputDataSource.getPos())+"], error: ["+e.getMessage()+"].", e);
}
finally
{
this.markUpdateEnd(inputDataSource.getPos());
}
}, executor);
}
catch (RejectedExecutionException ignore)
{
// can happen if the executor was shutdown while this task was queued
this.markUpdateEnd(inputDataSource.getPos());
return CompletableFuture.completedFuture(null);
}
}
/**
* After this method returns the inputData will be written to file.
*
* @param updatePos the position to update
*/
protected void updateDataSourceAtPos(long updatePos, @NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos)
{
boolean methodLocked = false;
// a lock is necessary to prevent two threads from writing to the same position at once,
// if that happens only the second update will apply and the LOD will end up with hole(s)
ReentrantLock updateLock = this.updateLockProvider.getLock(updatePos);
try
{
if (lockOnUpdatePos)
{
methodLocked = true;
updateLock.lock();
this.lockedPosSet.add(updatePos);
}
// get or create the data source
try (TDataSource recipientDataSource = this.get(updatePos))
{
if (recipientDataSource != null)
{
boolean dataModified = recipientDataSource.update(inputData, this.level);
if (dataModified)
{
// save the updated data to the database
try (TDTO dto = this.createDtoFromDataSource(recipientDataSource))
{
this.repo.save(dto);
}
for (IDataSourceUpdateFunc<TDataSource> listener : this.dateSourceUpdateListeners)
{
if (listener != null)
{
listener.OnDataSourceUpdated(recipientDataSource);
}
}
}
}
}
}
catch (Exception e)
{
LOGGER.error("Error updating pos ["+DhSectionPos.toString(updatePos)+"], error: "+e.getMessage(), e);
}
finally
{
if (methodLocked)
{
updateLock.unlock();
this.lockedPosSet.remove(updatePos);
}
}
}
//================//
// helper methods //
//================//
/** used for debugging to track which positions are queued for updating */
private void markUpdateStart(long dataSourcePos)
{
this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) ->
{
if (atomicCount == null)
{
atomicCount = new AtomicInteger(0);
}
atomicCount.incrementAndGet();
return atomicCount;
});
}
/** used for debugging to track which positions are queued for updating */
private void markUpdateEnd(long dataSourcePos)
{
this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) ->
{
if (atomicCount != null && atomicCount.decrementAndGet() <= 0)
{
atomicCount = null;
}
return atomicCount;
});
}
//=========//
// cleanup //
//=========//
@Override
public void close()
{
try
{
this.closeLock.lock();
this.isShutdown = true;
// wait a moment so any queued saves can finish queuing,
// otherwise we might not see everything that needs saving and attempt to use a closed repo
Thread.sleep(200);
LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "].");
this.repo.close();
}
catch (InterruptedException ignore) { }
finally
{
this.closeLock.unlock();
}
}
//================//
// helper classes //
//================//
@FunctionalInterface
public interface IDataSourceUpdateFunc<TDataSource>
{
void OnDataSourceUpdated(TDataSource updatedFullDataSource);
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java
-36
View File
@@ -1,36 +0,0 @@
package com.seibel.distanthorizons.core.file;
import com.seibel.distanthorizons.api.enums.EDhApiDetailLevel;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.sql.dto.IBaseDTO;
/**
* Base for all data sources. <br><br>
*
* AutoCloseable Can be implemented to allow for disposing of pooled data sources. <br><br>
*
* @param <TDhLevel> there are times when we need specifically a client level vs a more generic level
*/
public interface IDataSource<TDhLevel extends IDhLevel> extends IBaseDTO<Long>, AutoCloseable
{
long getPos();
/** @return true if the data was changed */
boolean update(FullDataSourceV2 chunkData, TDhLevel level);
//===========//
// meta data //
//===========//
/**
* Returns the detail level of the data contained by this data source.
* IE: 0 for block, 1 for 2x2 blocks, etc.
*
* @see EDhApiDetailLevel
*/
byte getDataDetailLevel();
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java
+1 -1
View File
@@ -104,7 +104,7 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable
}
// write the new data into memory
memoryDataSource.update(inputDataSource);
memoryDataSource.updateFromChunk(inputDataSource);
// keep track of when the last time we saved something was
pair.updateLastWrittenTimestamp();
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java
-814
View File
@@ -1,814 +0,0 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.fullDatafile;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.api.internal.ClientApi;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import com.seibel.distanthorizons.core.logging.DhLogger;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.text.NumberFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
/**
* Handles reading/writing {@link FullDataSourceV2}
* to and from the database.
*/
public class FullDataSourceProviderV2
extends AbstractDataSourceHandler<FullDataSourceV2, FullDataSourceV2DTO, FullDataSourceV2Repo, IDhLevel>
implements IDebugRenderable
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 5;
/** how many parent update tasks can be in the queue at once */
protected static int getMaxUpdateTaskCount() { return NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD* Config.Common.MultiThreading.numberOfThreads.get(); }
/** indicates how long the update queue thread should wait between queuing ticks */
protected static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250;
/** how many data sources should be pulled down for migration at once */
private static final int MIGRATION_BATCH_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD;
/**
* 5 minutes <br>
* This should be much longer than any update should take. This is just
* to make sure the thread doesn't get stuck.
*/
private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000;
/**
* Interrupting the migration thread pool doesn't work well and may corrupt the database
* vs gracefully shutting down the thread ourselves.
*/
protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true);
protected final FullDataSourceProviderV1<IDhLevel> legacyFileHandler;
protected boolean migrationStartMessageQueued = false;
protected long legacyDeletionCount = -1;
protected long migrationCount = -1;
protected boolean migrationStoppedWithError = false;
/**
* Tracks which positions are currently being updated
* to prevent duplicate concurrent updates.
*/
public final Set<Long> updatingPosSet = ConcurrentHashMap.newKeySet();
// TODO only run thread if modifications happened recently
/**
* This isn't in {@link AbstractDataSourceHandler} since we only want to update
* the newest version of the full data, so if we have providers for either
* render data or old full data, we don't want to update them. <br><br>
*
* Will be null on the dedicated server since updates don't need to be propagated,
* only the highest detail level is needed.
*/
@Nullable
private final ThreadPoolExecutor updateQueueProcessor;
//=============//
// constructor //
//=============//
public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure) { this(level, saveStructure, null); }
public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride)
{
super(level, saveStructure, saveDirOverride);
this.legacyFileHandler = new FullDataSourceProviderV1<>(level, saveStructure, saveDirOverride);
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus);
String levelId = level.getLevelWrapper().getDhIdentifier();
// start migrating any legacy data sources present in the background
ThreadPoolExecutor executor = ThreadPoolUtil.getFullDataMigrationExecutor();
if (executor != null)
{
executor.execute(this::convertLegacyDataSources);
}
else
{
// shouldn't happen, but just in case
LOGGER.error("Unable to start migration for level: ["+levelId+"] due to missing executor.");
}
// update propagation doesn't need to be run on the server since only the highest detail level is needed
this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Parent Update Queue [" + levelId + "]");
this.updateQueueProcessor.execute(this::runUpdateQueue);
}
//====================//
// Abstract overrides //
//====================//
@Override
protected FullDataSourceV2Repo createRepo()
{
try
{
return new FullDataSourceV2Repo(AbstractDhRepo.DEFAULT_DATABASE_TYPE, new File(this.saveDir.getPath() + File.separator + ISaveStructure.DATABASE_NAME));
}
catch (SQLException e)
{
// should only happen if there is an issue with the database (it's locked or the folder path is missing)
// or the database update failed
throw new RuntimeException(e);
}
}
@Override
protected FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource)
{
try
{
// when creating new data use the compressor currently selected in the config
EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get();
return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum);
}
catch (IOException e)
{
LOGGER.warn("Unable to create DTO, error: "+e.getMessage(), e);
return null;
}
}
@Override
protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException
{ return dto.createDataSource(this.level.getLevelWrapper()); }
@Override
protected FullDataSourceV2 makeEmptyDataSource(long pos)
{ return FullDataSourceV2.createEmpty(pos); }
//================//
// parent updates //
//================//
private void runUpdateQueue()
{
while (!Thread.interrupted())
{
try
{
Thread.sleep(UPDATE_QUEUE_THREAD_DELAY_IN_MS);
PriorityTaskPicker.Executor executor = ThreadPoolUtil.getUpdatePropagatorExecutor();
if (executor == null || executor.isTerminated())
{
continue;
}
// TODO it might be worth skipping this logic if no parent updates happened
// update positions closest to the player (if not on a server)
// to make world gen appear faster
DhBlockPos targetBlockPos = DhBlockPos.ZERO;
if (MC_CLIENT != null && MC_CLIENT.playerExists())
{
targetBlockPos = MC_CLIENT.getPlayerBlockPos();
}
this.runParentUpdates(executor, targetBlockPos);
if (Config.Common.LodBuilding.Experimental.upsampleLowerDetailLodsToFillHoles.get())
{
this.runChildUpdates(executor, targetBlockPos);
}
}
catch (InterruptedException ignored)
{
Thread.currentThread().interrupt();
}
catch (Exception e)
{
LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e);
}
}
LOGGER.info("Update thread ["+Thread.currentThread().getName()+"] terminated.");
}
/** will always apply updates */
private void runParentUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos)
{
int maxUpdateTaskCount = getMaxUpdateTaskCount();
// queue parent updates
if (executor.getQueueSize() < maxUpdateTaskCount
&& this.updatingPosSet.size() < maxUpdateTaskCount)
{
// get the positions that need to be applied to their parents
LongArrayList parentUpdatePosList = this.repo.getPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount);
// combine updates together based on their parent
HashMap<Long, HashSet<Long>> updatePosByParentPos = new HashMap<>();
for (Long pos : parentUpdatePosList)
{
updatePosByParentPos.compute(DhSectionPos.getParentPos(pos), (parentPos, updatePosSet) ->
{
if (updatePosSet == null)
{
updatePosSet = new HashSet<>();
}
updatePosSet.add(pos);
return updatePosSet;
});
}
// queue the updates
for (Long parentUpdatePos : updatePosByParentPos.keySet())
{
// stop if there are already a bunch of updates queued
if (this.updatingPosSet.size() > maxUpdateTaskCount
|| executor.getQueueSize() > maxUpdateTaskCount
|| !this.updatingPosSet.add(parentUpdatePos))
{
break;
}
try
{
executor.execute(() ->
{
ReentrantLock parentWriteLock = this.updateLockProvider.getLock(parentUpdatePos);
boolean parentLocked = false;
try
{
//LOGGER.info("updating parent: "+parentUpdatePos);
// Locking the parent before the children should prevent deadlocks.
// TryLock is used instead of lock so this thread can handle a different update.
if (parentWriteLock.tryLock())
{
parentLocked = true;
this.lockedPosSet.add(parentUpdatePos);
try (FullDataSourceV2 parentDataSource = this.get(parentUpdatePos))
{
// will return null if the file handler is shutting down
if (parentDataSource != null)
{
// apply each child pos to the parent
for (Long childPos : updatePosByParentPos.get(parentUpdatePos))
{
ReentrantLock childReadLock = this.updateLockProvider.getLock(childPos);
try
{
childReadLock.lock();
this.lockedPosSet.add(childPos);
try (FullDataSourceV2 childDataSource = this.get(childPos))
{
// can return null when the file handler is being shut down
if (childDataSource != null)
{
parentDataSource.update(childDataSource);
}
}
}
catch (Exception e)
{
LOGGER.error("Unexpected in parent update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e);
}
finally
{
childReadLock.unlock();
this.lockedPosSet.remove(childPos);
}
}
if (DhSectionPos.getDetailLevel(parentUpdatePos) < TOP_SECTION_DETAIL_LEVEL)
{
parentDataSource.applyToParent = true;
}
this.updateDataSourceAtPos(parentUpdatePos, parentDataSource, false);
for (Long childPos : updatePosByParentPos.get(parentUpdatePos))
{
this.repo.setApplyToParent(childPos, false);
}
}
}
}
}
finally
{
if (parentLocked)
{
parentWriteLock.unlock();
this.lockedPosSet.remove(parentUpdatePos);
}
this.updatingPosSet.remove(parentUpdatePos);
}
});
}
catch (RejectedExecutionException ignore)
{ /* the executor was shut down, it should be back up shortly and able to accept new jobs */ }
catch (Exception e)
{
this.updatingPosSet.remove(parentUpdatePos);
throw e;
}
}
}
}
/** stops if it finds any LOD data */
private void runChildUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos)
{
int maxUpdateTaskCount = getMaxUpdateTaskCount();
// queue child updates
if (executor.getQueueSize() < maxUpdateTaskCount
&& this.updatingPosSet.size() < maxUpdateTaskCount)
{
// get the positions that need to be applied to their children
LongArrayList childUpdatePosList = this.repo.getChildPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount);
// queue the updates
for (long parentUpdatePos : childUpdatePosList)
{
// stop if there are already a bunch of updates queued
if (this.updatingPosSet.size() > maxUpdateTaskCount
|| executor.getQueueSize() > maxUpdateTaskCount)
{
break;
}
// skip already updating positions
if (!this.updatingPosSet.add(parentUpdatePos))
{
continue;
}
try
{
executor.execute(() ->
{
ReentrantLock parentReadLock = this.updateLockProvider.getLock(parentUpdatePos);
boolean parentLocked = false;
try
{
//LOGGER.info("updating parent: "+parentUpdatePos);
// Locking the parent before the children should prevent deadlocks.
// TryLock is used instead of lock so this thread can handle a different update.
if (parentReadLock.tryLock())
{
parentLocked = true;
this.lockedPosSet.add(parentUpdatePos);
try (FullDataSourceV2 parentDataSource = this.get(parentUpdatePos))
{
// will return null if the file handler is shutting down
if (parentDataSource != null)
{
// apply parent to each child
for (int i = 0; i < 4; i++)
{
long childPos = DhSectionPos.getChildByIndex(parentUpdatePos, i);
ReentrantLock childWriteLock = this.updateLockProvider.getLock(childPos);
try
{
childWriteLock.lock();
this.lockedPosSet.add(childPos);
try (FullDataSourceV2 childDataSource = this.get(childPos))
{
// will return null if the file handler is shutting down
if (childDataSource != null)
{
childDataSource.update(parentDataSource);
// don't propagate child updates past the bottom of the tree
if (DhSectionPos.getDetailLevel(childPos) != DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL)
{
childDataSource.applyToChildren = true;
}
this.updateDataSourceAtPos(childPos, childDataSource, false);
}
}
}
catch (Exception e)
{
LOGGER.error("Unexpected in child update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e);
}
finally
{
childWriteLock.unlock();
this.lockedPosSet.remove(childPos);
}
}
this.repo.setApplyToChild(parentUpdatePos, false);
}
}
}
}
finally
{
if (parentLocked)
{
parentReadLock.unlock();
this.lockedPosSet.remove(parentUpdatePos);
}
this.updatingPosSet.remove(parentUpdatePos);
}
});
}
catch (RejectedExecutionException ignore)
{ /* the executor was shut down, it should be back up shortly and able to accept new jobs */ }
catch (Exception e)
{
this.updatingPosSet.remove(parentUpdatePos);
throw e;
}
}
}
}
//=======================//
// data source migration //
//=======================//
private void convertLegacyDataSources()
{
try
{
String levelId = this.level.getLevelWrapper().getDhIdentifier();
LOGGER.info("Attempting to migrate data sources for: [" + levelId + "]-[" + this.saveDir + "]...");
this.migrationThreadRunning.set(true);
//============================//
// delete unused data sources //
//============================//
// this could be done all at once via SQL,
// but doing it in chunks prevents locking the database for long periods of time
long unusedCount = 0;
long totalDeleteCount = this.legacyFileHandler.repo.getUnusedDataSourceCount();
if (totalDeleteCount != 0)
{
// this should only be shown once per session but should be shown during
// either when the deletion or migration phases start
this.showMigrationStartMessage();
LOGGER.info("deleting [" + levelId + "] - [" + totalDeleteCount + "] unused data sources...");
this.legacyDeletionCount = totalDeleteCount;
ArrayList<String> unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50);
while (unusedDataPosList.size() != 0)
{
unusedCount += unusedDataPosList.size();
this.legacyDeletionCount -= unusedDataPosList.size();
long startTime = System.currentTimeMillis();
// delete batch and get next batch
this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList);
unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50);
long endStart = System.currentTimeMillis();
long deleteTime = endStart - startTime;
LOGGER.info("Deleting [" + levelId + "] - [" + unusedCount + "/" + totalDeleteCount + "] in [" + deleteTime + "]ms ...");
// a slight delay is added to prevent accidentally locking the database when deleting a lot of rows
// (that shouldn't be the case since we're using WAL journaling, but just in case)
try
{
// use the delete time so we don't make powerful computers wait super long
// and weak computers wait no time at all
Thread.sleep(deleteTime / 2);
}
catch (InterruptedException ignore)
{
}
}
LOGGER.info("Done deleting [" + levelId + "] - [" + totalDeleteCount + "] unused data sources.");
}
//===========//
// migration //
//===========//
long totalMigrationCount = this.legacyFileHandler.getDataSourceMigrationCount();
this.migrationCount = totalMigrationCount;
LOGGER.info("Found [" + totalMigrationCount + "] data sources that need migration.");
ArrayList<FullDataSourceV1> legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT);
if (!legacyDataSourceList.isEmpty())
{
this.showMigrationStartMessage();
try
{
// keep going until every data source has been migrated
int progressCount = 0;
while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get())
{
NumberFormat numFormat = F3Screen.NUMBER_FORMAT;
LOGGER.info("Migrating [" + levelId + "] - [" + numFormat.format(progressCount) + "/" + numFormat.format(totalMigrationCount) + "]...");
ArrayList<CompletableFuture<Void>> updateFutureList = new ArrayList<>();
for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++)
{
FullDataSourceV1 legacyDataSource = legacyDataSourceList.get(i);
try
{
// convert the legacy data source to the new format,
// this is a relatively cheap operation
FullDataSourceV2 newDataSource = FullDataSourceV2.createFromLegacyDataSourceV1(legacyDataSource);
newDataSource.applyToParent = true;
// the actual update process can be moderately expensive due to having to update
// the render data along with the full data, so running it async on the update threads gains us a good bit of speed
CompletableFuture<Void> future = this.updateDataSourceAsync(newDataSource);
updateFutureList.add(future);
future.thenRun(() ->
{
// after the update finishes the legacy data source can be safely deleted
this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos());
newDataSource.close();
});
}
catch (Exception e)
{
Long migrationPos = legacyDataSource.getPos();
LOGGER.warn("Unexpected issue migrating data source at pos [" + DhSectionPos.toString(migrationPos) + "]. Error: " + e.getMessage(), e);
this.legacyFileHandler.markMigrationFailed(migrationPos);
}
}
try
{
// wait for each thread to finish updating
CompletableFuture<Void> combinedFutures = CompletableFuture.allOf(updateFutureList.toArray(new CompletableFuture[0]));
combinedFutures.get(MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS);
}
catch (InterruptedException | TimeoutException e)
{
LOGGER.warn("Migration update timed out after [" + MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS + "] milliseconds. Migration will re-try the same positions again in a moment.", e);
}
catch (ExecutionException e)
{
LOGGER.warn("Migration update failed. Migration will re-try the same positions again. Error:" + e.getMessage(), e);
}
legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT);
progressCount += legacyDataSourceList.size();
this.migrationCount -= legacyDataSourceList.size();
}
}
catch (Exception e)
{
LOGGER.info("migration stopped due to error for: [" + levelId + "]-[" + this.saveDir + "], error: [" + e.getMessage() + "].", e);
this.showMigrationEndMessage(false);
this.migrationStoppedWithError = true;
}
finally
{
if (this.migrationThreadRunning.get())
{
LOGGER.info("migration complete for: [" + levelId + "]-[" + this.saveDir + "].");
this.showMigrationEndMessage(true);
this.migrationCount = 0;
}
else
{
LOGGER.info("migration stopped for: [" + levelId + "]-[" + this.saveDir + "].");
this.showMigrationEndMessage(false);
this.migrationStoppedWithError = true;
}
}
}
else
{
LOGGER.info("No migration necessary.");
}
}
finally
{
this.migrationThreadRunning.set(false);
}
}
public long getLegacyDeletionCount() { return this.legacyDeletionCount; }
public long getTotalMigrationCount() { return this.migrationCount; }
public boolean getMigrationStoppedWithError() { return this.migrationStoppedWithError; }
private void showMigrationStartMessage()
{
if (this.migrationStartMessageQueued)
{
return;
}
this.migrationStartMessageQueued = true;
String levelId = this.level.getLevelWrapper().getDhIdentifier();
ClientApi.INSTANCE.showChatMessageNextFrame(
"Old Distant Horizons data is being migrated for ["+levelId+"]. \n" +
"While migrating LODs may load slowly \n" +
"and DH world gen will be disabled. \n" +
"You can see migration progress in the F3 menu."
);
}
private void showMigrationEndMessage(boolean success)
{
String levelId = this.level.getLevelWrapper().getDhIdentifier();
if (success)
{
ClientApi.INSTANCE.showChatMessageNextFrame("Distant Horizons data migration for ["+levelId+"] completed.");
}
else
{
ClientApi.INSTANCE.showChatMessageNextFrame(
"Distant Horizons data migration for ["+levelId+"] stopped. \n" +
"Some data may not have been migrated."
);
}
}
//=======================//
// retrieval (world gen) //
//=======================//
/**
* Returns true if this provider can generate or retrieve
* {@link FullDataSourceV2}'s that aren't currently in the database.
*/
public boolean canRetrieveMissingDataSources()
{
// the base handler just handles basic reading/writing
// to the database and as such can't retrieve anything else.
return false;
}
/**
* Returns false if this provider isn't accepting new requests,
* this can be due to having a full queue or some other
* limiting factor. <br><br>
*
* Note: when overriding make sure to add: <br>
* <code>
* if (!super.canQueueRetrieval()) <br>
* { <br>
* return false; <br>
* } <br>
* </code>
* to the beginning of your override.
* Otherwise, parent retrieval limits will be ignored.
*/
public boolean canQueueRetrieval()
{
// Retrieval shouldn't happen while an unknown number of
// legacy data sources are present.
// If retrieval was allowed we might run into concurrency issues.
return !this.migrationThreadRunning.get();
}
/**
* @return null if this provider can't generate any positions and
* an empty array if all positions were generated
*/
@Nullable
public LongArrayList getPositionsToRetrieve(Long pos) { return null; }
/** @return true if the position was queued, false if not */
@Nullable
public CompletableFuture<WorldGenResult> queuePositionForRetrieval(Long genPos) { return null; }
/** does nothing if the given position isn't present in the queue */
public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf) { }
public void clearRetrievalQueue() { }
/** Can be used to display how many total retrieval requests might be available. */
public void setTotalRetrievalPositionCount(int newCount) { }
/** Can be used to display how many total chunk retrieval requests should be available. */
public void setEstimatedRemainingRetrievalChunkCount(int newCount) { }
public boolean fileExists(long pos) { return this.repo.getDataSizeInBytes(pos) > 0; }
//========================//
// multiplayer networking //
//========================//
@Nullable
public Long getTimestampForPos(long pos)
{ return this.repo.getTimestampForPos(pos); }
//===========//
// overrides //
//===========//
@Override
public void debugRender(DebugRenderer renderer)
{
this.lockedPosSet
.forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 74f, 0.15f, Color.PINK)); });
this.queuedUpdateCountsByPos
.forEach((pos, updateCountRef) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f + (updateCountRef.get() * 16f), 0.20f, Color.WHITE)); });
this.updatingPosSet
.forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f, 0.20f, Color.MAGENTA)); });
}
@Override
public void close()
{
super.close();
if (this.updateQueueProcessor != null)
{
this.updateQueueProcessor.shutdownNow();
}
this.legacyFileHandler.close();
this.migrationThreadRunning.set(false);
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java
+15 -16
View File
@@ -23,6 +23,8 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio
import com.seibel.distanthorizons.core.api.internal.SharedApi;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataUpdatePropagatorV2;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.generation.DhLightingEngine;
import com.seibel.distanthorizons.core.generation.IFullDataSourceRetrievalQueue;
@@ -41,7 +43,6 @@ import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import com.seibel.distanthorizons.core.logging.DhLogger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -80,7 +81,16 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
//=============//
public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure) { super(level, saveStructure); }
public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); }
public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride)
{
super(level, saveStructure, saveDirOverride);
this.addDataSourceUpdateListener((@NotNull FullDataSourceV2 updatedData) ->
{
this.onWorldGenTaskComplete(WorldGenResult.CreateSuccess(updatedData.getPos()), null);
});
}
@@ -177,7 +187,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
{
boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue);
LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!");
LOGGER.info("Set world gen queue for level [" + this.level.getLevelWrapper().getDhIdentifier() + "].");
LOGGER.info("Set world gen queue for level [" + this.levelId + "].");
}
@Override
@@ -213,7 +223,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
PriorityTaskPicker.Executor renderLoadExecutor = ThreadPoolUtil.getRenderLoadingExecutor();
if (renderLoadExecutor == null
|| renderLoadExecutor.getQueueSize() >= getMaxUpdateTaskCount() / 2)
|| renderLoadExecutor.getQueueSize() >= FullDataUpdatePropagatorV2.getMaxPropagateTaskCount() / 2)
{
// don't queue additional world gen requests if the render loader handler is overwhelmed,
// otherwise LODs may not load in properly
@@ -222,7 +232,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
PriorityTaskPicker.Executor fileHandlerExecutor = ThreadPoolUtil.getFileHandlerExecutor();
if (fileHandlerExecutor == null
|| fileHandlerExecutor.getQueueSize() >= getMaxUpdateTaskCount() / 2)
|| fileHandlerExecutor.getQueueSize() >= FullDataUpdatePropagatorV2.getMaxPropagateTaskCount() / 2)
{
// don't queue additional world gen requests if the file handler is overwhelmed,
// otherwise LODs may not load in properly
@@ -294,17 +304,6 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
return worldGenFuture;
}
@Override
protected void updateDataSourceAtPos(long updatePos, @NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos)
{
super.updateDataSourceAtPos(updatePos, inputData, lockOnUpdatePos);
//if (SharedApi.getEnvironment() != EWorldEnvironment.CLIENT_ONLY)
// LOGGER.info("updated ["+DhSectionPos.toString(updatePos)+"]");
this.onWorldGenTaskComplete(WorldGenResult.CreateSuccess(updatePos), null);
}
@Override
public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf)
{
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IDataSourceUpdateListenerFunc.java
+7
View File
@@ -0,0 +1,7 @@
package com.seibel.distanthorizons.core.file.fullDatafile;
@FunctionalInterface
public interface IDataSourceUpdateListenerFunc<TDataSource>
{
void OnDataSourceUpdated(TDataSource updatedFullDataSource);
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java → core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java
+3 -3
View File
@@ -1,4 +1,4 @@
package com.seibel.distanthorizons.core.file.fullDatafile;
package com.seibel.distanthorizons.core.file.fullDatafile.V1;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
@@ -43,10 +43,10 @@ public class FullDataSourceProviderV1<TDhLevel extends IDhLevel>
// constructor //
//=============//
public FullDataSourceProviderV1(TDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride)
public FullDataSourceProviderV1(TDhLevel level, File saveDir)
{
this.level = level;
this.saveDir = (saveDirOverride == null) ? saveStructure.getSaveFolder(level.getLevelWrapper()) : saveDirOverride;
this.saveDir = saveDir;
if (!this.saveDir.exists() && !this.saveDir.mkdirs())
{
LOGGER.warn("Unable to create full data folder, file saving may fail.");
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/DataMigratorV1.java
+346
View File
@@ -0,0 +1,346 @@
package com.seibel.distanthorizons.core.file.fullDatafile.V2;
import com.seibel.distanthorizons.core.api.internal.ClientApi;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V1.FullDataSourceProviderV1;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import java.io.File;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
public class DataMigratorV1 implements IDebugRenderable, AutoCloseable
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
/** how many data sources should be pulled down for migration at once */
private static final int MIGRATION_BATCH_COUNT = FullDataUpdatePropagatorV2.NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD;
/**
* 5 minutes <br>
* This should be much longer than any update should take. This is just
* to make sure the thread doesn't get stuck.
*/
private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000;
private final FullDataUpdaterV2 dataUpdater;
private boolean migrationStartMessageQueued = false;
private long legacyDeletionCount = -1;
private long migrationCount = -1;
private boolean migrationStoppedWithError = false;
/**
* Interrupting the migration thread pool doesn't work well and may corrupt the database
* vs gracefully shutting down the thread ourselves.
*/
public final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true);
private final FullDataSourceProviderV1<IDhLevel> v1DataSourceProvider;
private final String levelId;
private final File saveDir;
//=============//
// constructor //
//=============//
public DataMigratorV1(
FullDataUpdaterV2 dataUpdater,
IDhLevel level, String levelId, File saveDir)
{
this.dataUpdater = dataUpdater;
this.saveDir = saveDir;
this.v1DataSourceProvider = new FullDataSourceProviderV1<>(level, saveDir);
this.levelId = levelId;
// start migrating any legacy data sources present in the background
ThreadPoolExecutor executor = ThreadPoolUtil.getFullDataMigrationExecutor();
if (executor != null)
{
executor.execute(this::convertLegacyDataSources);
}
else
{
// shouldn't happen, but just in case
LOGGER.error("Unable to start migration for level: ["+this.levelId+"] due to missing executor.");
}
}
//=======================//
// data source migration //
//=======================//
private void convertLegacyDataSources()
{
try
{
LOGGER.debug("Attempting to migrate data sources for: [" + this.levelId + "]-[" + this.saveDir + "]...");
this.migrationThreadRunning.set(true);
//============================//
// delete unused data sources //
//============================//
// this could be done all at once via SQL,
// but doing it in chunks prevents locking the database for long periods of time
long unusedCount = 0;
long totalDeleteCount = this.v1DataSourceProvider.repo.getUnusedDataSourceCount();
if (totalDeleteCount != 0)
{
// this should only be shown once per session but should be shown during
// either when the deletion or migration phases start
this.showMigrationStartMessage();
LOGGER.info("deleting [" + this.levelId + "] - [" + totalDeleteCount + "] unused data sources...");
this.legacyDeletionCount = totalDeleteCount;
ArrayList<String> unusedDataPosList = this.v1DataSourceProvider.repo.getUnusedDataSourcePositionStringList(50);
while (unusedDataPosList.size() != 0)
{
unusedCount += unusedDataPosList.size();
this.legacyDeletionCount -= unusedDataPosList.size();
long startTime = System.currentTimeMillis();
// delete batch and get next batch
this.v1DataSourceProvider.repo.deleteUnusedLegacyData(unusedDataPosList);
unusedDataPosList = this.v1DataSourceProvider.repo.getUnusedDataSourcePositionStringList(50);
long endStart = System.currentTimeMillis();
long deleteTime = endStart - startTime;
LOGGER.info("Deleting [" + this.levelId + "] - [" + unusedCount + "/" + totalDeleteCount + "] in [" + deleteTime + "]ms ...");
// a slight delay is added to prevent accidentally locking the database when deleting a lot of rows
// (that shouldn't be the case since we're using WAL journaling, but just in case)
try
{
// use the delete time so we don't make powerful computers wait super long
// and weak computers wait no time at all
Thread.sleep(deleteTime / 2);
}
catch (InterruptedException ignore)
{
}
}
LOGGER.info("Done deleting [" + this.levelId + "] - [" + totalDeleteCount + "] unused data sources.");
}
//===========//
// migration //
//===========//
long totalMigrationCount = this.v1DataSourceProvider.getDataSourceMigrationCount();
this.migrationCount = totalMigrationCount;
LOGGER.debug("Found [" + totalMigrationCount + "] data sources that need migration.");
ArrayList<FullDataSourceV1> legacyDataSourceList = this.v1DataSourceProvider.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT);
if (!legacyDataSourceList.isEmpty())
{
this.showMigrationStartMessage();
try
{
// keep going until every data source has been migrated
int progressCount = 0;
while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get())
{
NumberFormat numFormat = F3Screen.NUMBER_FORMAT;
LOGGER.info("Migrating [" + this.levelId + "] - [" + numFormat.format(progressCount) + "/" + numFormat.format(totalMigrationCount) + "]...");
ArrayList<CompletableFuture<Void>> updateFutureList = new ArrayList<>();
for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++)
{
FullDataSourceV1 legacyDataSource = legacyDataSourceList.get(i);
try
{
// convert the legacy data source to the new format,
// this is a relatively cheap operation
FullDataSourceV2 newDataSource = FullDataSourceV2.createFromLegacyDataSourceV1(legacyDataSource);
newDataSource.applyToParent = true;
// the actual update process can be moderately expensive due to having to update
// the render data along with the full data, so running it async on the update threads gains us a good bit of speed
CompletableFuture<Void> future = this.dataUpdater.updateDataSourceAsync(newDataSource);
updateFutureList.add(future);
future.thenRun(() ->
{
// after the update finishes the legacy data source can be safely deleted
this.v1DataSourceProvider.repo.deleteWithKey(legacyDataSource.getPos());
newDataSource.close();
});
}
catch (Exception e)
{
long migrationPos = legacyDataSource.getPos();
LOGGER.warn("Unexpected issue migrating data source at pos [" + DhSectionPos.toString(migrationPos) + "]. Error: " + e.getMessage(), e);
this.v1DataSourceProvider.markMigrationFailed(migrationPos);
}
}
try
{
// wait for each thread to finish updating
CompletableFuture<Void> combinedFutures = CompletableFuture.allOf(updateFutureList.toArray(new CompletableFuture[0]));
combinedFutures.get(MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS);
}
catch (InterruptedException | TimeoutException e)
{
LOGGER.warn("Migration update timed out after [" + MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS + "] milliseconds. Migration will re-try the same positions again in a moment.", e);
}
catch (ExecutionException e)
{
LOGGER.warn("Migration update failed. Migration will re-try the same positions again. Error:" + e.getMessage(), e);
}
legacyDataSourceList = this.v1DataSourceProvider.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT);
progressCount += legacyDataSourceList.size();
this.migrationCount -= legacyDataSourceList.size();
}
}
catch (Exception e)
{
LOGGER.info("migration stopped due to error for: [" + this.levelId + "]-[" + this.saveDir + "], error: [" + e.getMessage() + "].", e);
this.showMigrationEndMessage(false);
this.migrationStoppedWithError = true;
}
finally
{
if (this.migrationThreadRunning.get())
{
LOGGER.info("migration complete for: [" + this.levelId + "]-[" + this.saveDir + "].");
this.showMigrationEndMessage(true);
this.migrationCount = 0;
}
else
{
LOGGER.info("migration stopped for: [" + this.levelId + "]-[" + this.saveDir + "].");
this.showMigrationEndMessage(false);
this.migrationStoppedWithError = true;
}
}
}
else
{
LOGGER.info("No migration necessary.");
}
}
finally
{
this.migrationThreadRunning.set(false);
}
}
private void showMigrationStartMessage()
{
if (this.migrationStartMessageQueued)
{
return;
}
this.migrationStartMessageQueued = true;
ClientApi.INSTANCE.showChatMessageNextFrame(
"Old Distant Horizons data is being migrated for ["+this.levelId+"]. \n" +
"While migrating LODs may load slowly \n" +
"and DH world gen will be disabled. \n" +
"You can see migration progress in the F3 menu."
);
}
private void showMigrationEndMessage(boolean success)
{
if (success)
{
ClientApi.INSTANCE.showChatMessageNextFrame("Distant Horizons data migration for ["+this.levelId+"] completed.");
}
else
{
ClientApi.INSTANCE.showChatMessageNextFrame(
"Distant Horizons data migration for ["+this.levelId+"] stopped. \n" +
"Some data may not have been migrated."
);
}
}
//===========//
// debugging //
//===========//
public void addDebugMenuStringsToList(List<String> messageList)
{
// migration
boolean migrationErrored = this.migrationStoppedWithError;
if (!migrationErrored)
{
long legacyDeletionCount = this.legacyDeletionCount;
if (legacyDeletionCount > 0)
{
messageList.add(" Migrating - Deleting #: " + F3Screen.NUMBER_FORMAT.format(legacyDeletionCount));
}
long migrationCount = this.migrationCount;
if (migrationCount > 0)
{
messageList.add(" Migrating - Conversion #: " + F3Screen.NUMBER_FORMAT.format(migrationCount));
}
}
else
{
messageList.add(" Migration Failed");
}
}
//===========//
// overrides //
//===========//
@Override
public void debugRender(DebugRenderer renderer)
{
// nothing currently needed
}
@Override
public void close()
{
//LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "].");
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java
+452
View File
@@ -0,0 +1,452 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.seibel.distanthorizons.core.file.fullDatafile.V2;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo;
import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
/**
* Handles reading/writing {@link FullDataSourceV2}
* to and from the database.
*/
public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
private static final Set<String> CORRUPT_DATA_ERRORS_LOGGED = Collections.newSetFromMap(new ConcurrentHashMap<>());
/**
* The highest numerical detail level possible.
* Used when determining which positions to update.
*
* @see FullDataSourceProviderV2#LEAF_SECTION_DETAIL_LEVEL
*/
public static final byte ROOT_SECTION_DETAIL_LEVEL
= DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL
+ LodUtil.REGION_DETAIL_LEVEL;
/**
* The lowest numerical detail level possible.
*
* @see FullDataSourceProviderV2#ROOT_SECTION_DETAIL_LEVEL
*/
public static final byte LEAF_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
public final FullDataSourceV2Repo repo;
protected final AtomicBoolean isShutdownRef = new AtomicBoolean(false);
protected final File saveDir;
protected final IDhLevel level;
protected final String levelId;
private final FullDataUpdaterV2 dataUpdater;
private final FullDataUpdatePropagatorV2 updatePropagator;
private final DataMigratorV1 dataMigratorV1;
//=============//
// constructor //
//=============//
public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure) { this(level, saveStructure, null); }
public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride)
{
this.saveDir = (saveDirOverride == null) ? saveStructure.getSaveFolder(level.getLevelWrapper()) : saveDirOverride;
this.repo = this.createRepo();
this.level = level;
this.levelId = this.level.getLevelWrapper().getDhIdentifier();
this.dataUpdater = new FullDataUpdaterV2(this, this.levelId);
this.updatePropagator = new FullDataUpdatePropagatorV2(this, this.dataUpdater, this.levelId);
this.dataMigratorV1 = new DataMigratorV1(this.dataUpdater, this.level, this.levelId, this.saveDir);
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus);
}
private FullDataSourceV2Repo createRepo()
{
try
{
return new FullDataSourceV2Repo(AbstractDhRepo.DEFAULT_DATABASE_TYPE, new File(this.saveDir.getPath() + File.separator + ISaveStructure.DATABASE_NAME));
}
catch (SQLException e)
{
// should only happen if there is an issue with the database (it's locked or the folder path is missing)
// or the database update failed
throw new RuntimeException(e);
}
}
//=================//
// event listeners //
//=================//
public void addDataSourceUpdateListener(IDataSourceUpdateListenerFunc<FullDataSourceV2> listener)
{
synchronized (this.dataUpdater)
{
this.dataUpdater.dateSourceUpdateListeners.add(listener);
}
}
public void removeDataSourceUpdateListener(IDataSourceUpdateListenerFunc<FullDataSourceV2> listener)
{
synchronized (this.dataUpdater)
{
this.dataUpdater.dateSourceUpdateListeners.add(listener);
}
}
//================//
// DTO converters //
//================//
protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException
{ return dto.createDataSource(this.level.getLevelWrapper(), null); }
protected FullDataSourceV2 createAdjDataSourceFromDto(FullDataSourceV2DTO dto, EDhDirection direction) throws InterruptedException, IOException, DataCorruptedException
{ return dto.createDataSource(this.level.getLevelWrapper(), direction); }
//=========================//
// basic DataSource getter //
//=========================//
/**
* Returns the {@link FullDataSourceV2} for the given section position. <Br>
* The returned data source may be null if repo is in the process of shutting down. <Br> <Br>
*
* This call is concurrent. I.e. it supports being called by multiple threads at the same time.
*/
public CompletableFuture<FullDataSourceV2> getAsync(long pos)
{
if (this.isShutdownRef.get())
{
return CompletableFuture.completedFuture(null);
}
AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
try
{
return CompletableFuture.supplyAsync(() -> this.get(pos), executor);
}
catch (RejectedExecutionException ignore)
{
// the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back
return CompletableFuture.completedFuture(null);
}
}
/**
* Should only be used in internal file handler methods where we are already running on a file handler thread.
* Can return null if the repo is in the process of being shut down
* @see FullDataSourceProviderV2#getAsync(long)
*/
@Nullable
public FullDataSourceV2 get(long pos)
{
if (this.isShutdownRef.get())
{
return null;
}
try(FullDataSourceV2DTO dto = this.repo.getByKey(pos))
{
if (dto == null)
{
return FullDataSourceV2.createEmpty(pos);
}
try
{
FullDataSourceV2 dataSource = this.createDataSourceFromDto(dto);
// automatically create and save adjacent data if missing
if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA)
{
EDhApiDataCompressionMode compressionMode = Config.Common.LodBuilding.dataCompression.get();
try(FullDataSourceV2DTO updatedDto = FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionMode))
{
this.repo.save(updatedDto);
}
}
return dataSource;
}
catch (DataCorruptedException e)
{
this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e);
this.repo.deleteWithKey(pos);
}
}
catch (InterruptedException ignore) { }
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e);
}
// an error occurred
return null;
}
protected void tryLogCorruptedDataError(String whereClause, Exception e)
{
// there's a rare issue where the exception doesn't
// have a message, which can cause problems
String message = (e.getMessage() == null) ? e.getMessage() : "No Error message for exception ["+e.getClass().getSimpleName()+"]";
// Only log each message type once.
// This is done to prevent logging "No compression mode with the value [2]" 10,000 times
// if the user is migrating from a nightly build and used ZStd.
if (CORRUPT_DATA_ERRORS_LOGGED.add(message))
{
LOGGER.warn("Corrupted data found at [" + whereClause + "]. Data at will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + message + "].", e);
}
}
//=================//
// partial getters //
//=================//
/**
* Only returns the data row/column for the given compass-cardinal
* direction. <br>
* This is generally used for generating LOD render data
* where we only need the adjacent data, not the full thing.
*/
public FullDataSourceV2 getAdjForDirection(long pos, EDhDirection direction)
{
if (this.isShutdownRef.get())
{
return null;
}
try(FullDataSourceV2DTO dto = this.repo.getAdjByPosAndDirection(pos, direction))
{
if (dto == null)
{
return FullDataSourceV2.createEmpty(pos);
}
// migrate to the V2 format first if needed
if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA)
{
// get automatically converts from V1 to V2
FullDataSourceV2 migratedDataSource = this.get(pos);
if (migratedDataSource != null)
{
migratedDataSource.clearAllNonAdjData(direction);
}
return migratedDataSource;
}
try
{
// load from database
return this.createAdjDataSourceFromDto(dto, direction);
}
catch (DataCorruptedException e)
{
this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e);
this.repo.deleteWithKey(pos);
}
}
catch (InterruptedException ignore) { }
catch (IOException e)
{
LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e);
}
// an error occurred
return null;
}
//=======================//
// retrieval (world gen) //
//=======================//
/**
* Returns true if this provider can generate or retrieve
* {@link FullDataSourceV2}'s that aren't currently in the database.
*/
public boolean canRetrieveMissingDataSources()
{
// the base handler just handles basic reading/writing
// to the database and as such can't retrieve anything else.
return false;
}
/**
* Returns false if this provider isn't accepting new requests,
* this can be due to having a full queue or some other
* limiting factor. <br><br>
*
* Note: when overriding make sure to add: <br>
* <code>
* if (!super.canQueueRetrieval()) <br>
* { <br>
* return false; <br>
* } <br>
* </code>
* to the beginning of your override.
* Otherwise, parent retrieval limits will be ignored.
*/
public boolean canQueueRetrieval()
{
// Retrieval shouldn't happen while an unknown number of
// legacy data sources are present.
// If retrieval was allowed we might run into concurrency issues.
return !this.dataMigratorV1.migrationThreadRunning.get();
}
/**
* @return null if this provider can't generate any positions and
* an empty array if all positions were generated
*/
@Nullable
public LongArrayList getPositionsToRetrieve(Long pos) { return null; }
/** @return true if the position was queued, false if not */
@Nullable
public CompletableFuture<WorldGenResult> queuePositionForRetrieval(Long genPos) { return null; }
/** does nothing if the given position isn't present in the queue */
public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf) { }
public void clearRetrievalQueue() { }
/** Can be used to display how many total retrieval requests might be available. */
public void setTotalRetrievalPositionCount(int newCount) { }
/** Can be used to display how many total chunk retrieval requests should be available. */
public void setEstimatedRemainingRetrievalChunkCount(int newCount) { }
//=============//
// data update //
//=============//
public CompletableFuture<Void> updateDataSourceAsync(@NotNull FullDataSourceV2 inputData)
{ return this.dataUpdater.updateDataSourceAsync(inputData); }
//========================//
// multiplayer networking //
//========================//
@Nullable
public Long getTimestampForPos(long pos)
{
if (this.isShutdownRef.get())
{
return null;
}
return this.repo.getTimestampForPos(pos);
}
//===========//
// debugging //
//===========//
public void addDebugMenuStringsToList(List<String> messageList)
{
this.dataMigratorV1.addDebugMenuStringsToList(messageList);
}
//===========//
// overrides //
//===========//
@Override
public void debugRender(DebugRenderer renderer)
{
this.dataUpdater.debugRender(renderer);
this.updatePropagator.debugRender(renderer);
this.dataMigratorV1.debugRender(renderer);
}
@Override
public void close()
{
LOGGER.debug("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "].");
this.isShutdownRef.set(true);
this.dataUpdater.close();
this.updatePropagator.close();
this.dataMigratorV1.close();
this.repo.close();
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java
+399
View File
@@ -0,0 +1,399 @@
package com.seibel.distanthorizons.core.file.fullDatafile.V2;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.jetbrains.annotations.Nullable;
import java.awt.*;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseable
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
/** indicates how long the update queue thread should wait between queuing ticks */
protected static final int PROPAGATE_QUEUE_THREAD_DELAY_IN_MS = 250;
public static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 5;
/** how many parent update tasks can be in the queue at once */
public static int getMaxPropagateTaskCount()
{ return NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Common.MultiThreading.numberOfThreads.get(); }
/**
* Tracks which positions are currently being updated
* to prevent duplicate concurrent updates.
*/
private final Set<Long> updatingPosSet = ConcurrentHashMap.newKeySet();
// TODO only run thread if modifications happened recently
/**
* Will be null on the dedicated server since updates don't need to be propagated,
* only the highest detail level is needed.
*/
@Nullable
public final ThreadPoolExecutor updateQueueProcessor;
private final AtomicBoolean isShutdownRef = new AtomicBoolean(false);
private final String levelId;
private final FullDataSourceProviderV2 provider;
private final FullDataUpdaterV2 dataUpdater;
//=============//
// constructor //
//=============//
public FullDataUpdatePropagatorV2(FullDataSourceProviderV2 provider, FullDataUpdaterV2 dataUpdater, String levelId)
{
this.provider = provider;
this.dataUpdater = dataUpdater;
this.levelId = levelId;
// update propagation doesn't need to be run on the server since only the highest detail level is needed
this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Update Propagate Queue [" + this.levelId + "]");
this.updateQueueProcessor.execute(this::runUpdateQueue);
}
//================//
// parent updates //
//================//
private void runUpdateQueue()
{
while (!Thread.interrupted())
{
try
{
Thread.sleep(PROPAGATE_QUEUE_THREAD_DELAY_IN_MS);
PriorityTaskPicker.Executor executor = ThreadPoolUtil.getUpdatePropagatorExecutor();
if (executor == null || executor.isTerminated())
{
continue;
}
// TODO it might be worth skipping this logic if no parent updates happened
// update positions closest to the player (if not on a server)
// to make world gen appear faster
DhBlockPos targetBlockPos = DhBlockPos.ZERO;
if (MC_CLIENT != null
&& MC_CLIENT.playerExists())
{
targetBlockPos = MC_CLIENT.getPlayerBlockPos();
}
this.runParentUpdates(executor, targetBlockPos);
if (Config.Common.LodBuilding.Experimental.upsampleLowerDetailLodsToFillHoles.get())
{
this.runChildUpdates(executor, targetBlockPos);
}
}
catch (InterruptedException ignored)
{
Thread.currentThread().interrupt();
}
catch (Exception e)
{
LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e);
}
}
}
/** will always apply updates */
private void runParentUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos)
{
int maxUpdateTaskCount = getMaxPropagateTaskCount();
// queue parent updates
if (executor.getQueueSize() < maxUpdateTaskCount
&& this.updatingPosSet.size() < maxUpdateTaskCount)
{
// get the positions that need to be applied to their parents
LongArrayList parentUpdatePosList = this.provider.repo.getPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount);
// combine updates together based on their parent
HashMap<Long, HashSet<Long>> updatePosByParentPos = new HashMap<>();
for (Long pos : parentUpdatePosList)
{
updatePosByParentPos.compute(DhSectionPos.getParentPos(pos), (parentPos, updatePosSet) ->
{
if (updatePosSet == null)
{
updatePosSet = new HashSet<>();
}
updatePosSet.add(pos);
return updatePosSet;
});
}
// queue the updates
for (Long parentUpdatePos : updatePosByParentPos.keySet())
{
// stop if there are already a bunch of updates queued
if (this.updatingPosSet.size() > maxUpdateTaskCount
|| executor.getQueueSize() > maxUpdateTaskCount
|| !this.updatingPosSet.add(parentUpdatePos))
{
break;
}
try
{
executor.execute(() ->
{
ReentrantLock parentWriteLock = this.dataUpdater.updateLockProvider.getLock(parentUpdatePos);
boolean parentLocked = false;
try
{
//LOGGER.info("updating parent: "+parentUpdatePos);
// Locking the parent before the children should prevent deadlocks.
// TryLock is used instead of lock so this thread can handle a different update.
if (parentWriteLock.tryLock())
{
parentLocked = true;
this.dataUpdater.lockedPosSet.add(parentUpdatePos);
try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos))
{
// will return null if the file handler is shutting down
if (parentDataSource != null)
{
// apply each child pos to the parent
for (Long childPos : updatePosByParentPos.get(parentUpdatePos))
{
ReentrantLock childReadLock = this.dataUpdater.updateLockProvider.getLock(childPos);
try
{
childReadLock.lock();
this.dataUpdater.lockedPosSet.add(childPos);
try (FullDataSourceV2 childDataSource = this.provider.get(childPos))
{
// can return null when the file handler is being shut down
if (childDataSource != null)
{
parentDataSource.updateFromChunk(childDataSource);
}
}
}
catch (Exception e)
{
LOGGER.error("Unexpected in parent update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e);
}
finally
{
childReadLock.unlock();
this.dataUpdater.lockedPosSet.remove(childPos);
}
}
if (DhSectionPos.getDetailLevel(parentUpdatePos) < FullDataSourceProviderV2.ROOT_SECTION_DETAIL_LEVEL)
{
parentDataSource.applyToParent = true;
}
this.dataUpdater.updateDataSource(parentDataSource, false);
for (Long childPos : updatePosByParentPos.get(parentUpdatePos))
{
this.provider.repo.setApplyToParent(childPos, false);
}
}
}
}
}
finally
{
if (parentLocked)
{
parentWriteLock.unlock();
this.dataUpdater.lockedPosSet.remove(parentUpdatePos);
}
this.updatingPosSet.remove(parentUpdatePos);
}
});
}
catch (RejectedExecutionException ignore)
{ /* the executor was shut down, it should be back up shortly and able to accept new jobs */ }
catch (Exception e)
{
this.updatingPosSet.remove(parentUpdatePos);
throw e;
}
}
}
}
/** stops if it finds any LOD data */
private void runChildUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos)
{
int maxUpdateTaskCount = getMaxPropagateTaskCount();
// queue child updates
if (executor.getQueueSize() < maxUpdateTaskCount
&& this.updatingPosSet.size() < maxUpdateTaskCount)
{
// get the positions that need to be applied to their children
LongArrayList childUpdatePosList = this.provider.repo.getChildPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount);
// queue the updates
for (long parentUpdatePos : childUpdatePosList)
{
// stop if there are already a bunch of updates queued
if (this.updatingPosSet.size() > maxUpdateTaskCount
|| executor.getQueueSize() > maxUpdateTaskCount)
{
break;
}
// skip already updating positions
if (!this.updatingPosSet.add(parentUpdatePos))
{
continue;
}
try
{
executor.execute(() ->
{
ReentrantLock parentReadLock = this.dataUpdater.updateLockProvider.getLock(parentUpdatePos);
boolean parentLocked = false;
try
{
//LOGGER.info("updating parent: "+parentUpdatePos);
// Locking the parent before the children should prevent deadlocks.
// TryLock is used instead of lock so this thread can handle a different update.
if (parentReadLock.tryLock())
{
parentLocked = true;
this.dataUpdater.lockedPosSet.add(parentUpdatePos);
try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos))
{
// will return null if the file handler is shutting down
if (parentDataSource != null)
{
// apply parent to each child
for (int i = 0; i < 4; i++)
{
long childPos = DhSectionPos.getChildByIndex(parentUpdatePos, i);
ReentrantLock childWriteLock = this.dataUpdater.updateLockProvider.getLock(childPos);
try
{
childWriteLock.lock();
this.dataUpdater.lockedPosSet.add(childPos);
try (FullDataSourceV2 childDataSource = this.provider.get(childPos))
{
// will return null if the file handler is shutting down
if (childDataSource != null)
{
childDataSource.updateFromChunk(parentDataSource);
// don't propagate child updates past the bottom of the tree
if (DhSectionPos.getDetailLevel(childPos) != DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL)
{
childDataSource.applyToChildren = true;
}
this.dataUpdater.updateDataSource(childDataSource, false);
}
}
}
catch (Exception e)
{
LOGGER.error("Unexpected in child update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e);
}
finally
{
childWriteLock.unlock();
this.dataUpdater.lockedPosSet.remove(childPos);
}
}
this.provider.repo.setApplyToChild(parentUpdatePos, false);
}
}
}
}
finally
{
if (parentLocked)
{
parentReadLock.unlock();
this.dataUpdater.lockedPosSet.remove(parentUpdatePos);
}
this.updatingPosSet.remove(parentUpdatePos);
}
});
}
catch (RejectedExecutionException ignore)
{ /* the executor was shut down, it should be back up shortly and able to accept new jobs */ }
catch (Exception e)
{
this.updatingPosSet.remove(parentUpdatePos);
throw e;
}
}
}
}
//===========//
// overrides //
//===========//
@Override
public void debugRender(DebugRenderer renderer)
{
this.updatingPosSet
.forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f, 0.20f, Color.MAGENTA)); });
}
@Override
public void close()
{
if (this.updateQueueProcessor != null)
{
this.updateQueueProcessor.shutdownNow();
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java
+249
View File
@@ -0,0 +1,249 @@
package com.seibel.distanthorizons.core.file.fullDatafile.V2;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import org.jetbrains.annotations.NotNull;
import java.awt.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Set;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
protected final PositionalLockProvider updateLockProvider = new PositionalLockProvider();
/**
* generally just used for debugging,
* keeps track of which positions are currently locked.
*/
public final Set<Long> lockedPosSet = ConcurrentHashMap.newKeySet();
private final ConcurrentHashMap<Long, AtomicInteger> queuedUpdateCountsByPos = new ConcurrentHashMap<>();
public final ArrayList<IDataSourceUpdateListenerFunc<FullDataSourceV2>> dateSourceUpdateListeners = new ArrayList<>();
private final String levelId;
private final AtomicBoolean isShutdownRef = new AtomicBoolean(false);
private final FullDataSourceProviderV2 provider;
//=============//
// constructor //
//=============//
public FullDataUpdaterV2(FullDataSourceProviderV2 provider, String levelId)
{
this.provider = provider;
this.levelId = levelId;
}
//===============//
// data updating //
//===============//
/**
* Can be used if you don't want to lock the current thread
* Otherwise the sync version {@link FullDataUpdaterV2#updateDataSource(FullDataSourceV2, boolean)} may be a better choice.
*/
public CompletableFuture<Void> updateDataSourceAsync(@NotNull FullDataSourceV2 inputDataSource)
{
if (this.isShutdownRef.get())
{
return CompletableFuture.completedFuture(null);
}
AbstractExecutorService executor = ThreadPoolUtil.getChunkToLodBuilderExecutor();
if (executor == null || executor.isTerminated())
{
return CompletableFuture.completedFuture(null);
}
try
{
this.markUpdateStart(inputDataSource.getPos());
return CompletableFuture.runAsync(() ->
{
try
{
this.updateDataSource(inputDataSource, true);
}
catch (Exception e)
{
LOGGER.error("Unexpected error in async data source update at pos: ["+ DhSectionPos.toString(inputDataSource.getPos())+"], error: ["+e.getMessage()+"].", e);
}
finally
{
this.markUpdateEnd(inputDataSource.getPos());
}
}, executor);
}
catch (RejectedExecutionException ignore)
{
// can happen if the executor was shutdown while this task was queued
this.markUpdateEnd(inputDataSource.getPos());
return CompletableFuture.completedFuture(null);
}
}
/** After this method returns the inputData will be written to file. */
public void updateDataSource(@NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos)
{
if (this.isShutdownRef.get())
{
return;
}
long updatePos = inputData.getPos();
boolean methodLocked = false;
// a lock is necessary to prevent two threads from writing to the same position at once,
// if that happens only the second update will apply and the LOD will end up with hole(s)
ReentrantLock updateLock = this.updateLockProvider.getLock(updatePos);
try
{
if (lockOnUpdatePos)
{
methodLocked = true;
updateLock.lock();
this.lockedPosSet.add(updatePos);
}
// get or create the data source
try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos))
{
if (recipientDataSource != null)
{
boolean dataModified = recipientDataSource.updateFromChunk(inputData);
if (dataModified)
{
// save the updated data to the database
try (FullDataSourceV2DTO dto = this.createDtoFromDataSource(recipientDataSource))
{
if (dto != null)
{
this.provider.repo.save(dto);
}
}
for (IDataSourceUpdateListenerFunc<FullDataSourceV2> listener : this.dateSourceUpdateListeners)
{
if (listener != null)
{
listener.OnDataSourceUpdated(recipientDataSource);
}
}
}
}
}
}
catch (Exception e)
{
LOGGER.error("Error updating pos ["+DhSectionPos.toString(updatePos)+"], error: "+e.getMessage(), e);
}
finally
{
if (methodLocked)
{
updateLock.unlock();
this.lockedPosSet.remove(updatePos);
}
}
}
private FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource)
{
try
{
// when creating new data use the compressor currently selected in the config
EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get();
return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum);
}
catch (IOException e)
{
LOGGER.warn("Unable to create DTO, error: ["+e.getMessage() + "].", e);
return null;
}
}
//==================//
// debugger methods //
//==================//
/** used for debugging to track which positions are queued for updating */
private void markUpdateStart(long dataSourcePos)
{
this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) ->
{
if (atomicCount == null)
{
atomicCount = new AtomicInteger(0);
}
atomicCount.incrementAndGet();
return atomicCount;
});
}
/** used for debugging to track which positions are queued for updating */
private void markUpdateEnd(long dataSourcePos)
{
this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) ->
{
if (atomicCount != null && atomicCount.decrementAndGet() <= 0)
{
atomicCount = null;
}
return atomicCount;
});
}
//===========//
// overrides //
//===========//
@Override
public void debugRender(DebugRenderer renderer)
{
this.lockedPosSet
.forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 74f, 0.15f, Color.PINK)); });
this.queuedUpdateCountsByPos
.forEach((pos, updateCountRef) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f + (updateCountRef.get() * 16f), 0.20f, Color.WHITE)); });
}
@Override
public void close()
{
this.isShutdownRef.set(true);
}
}
core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java
+3 -3
View File
@@ -314,7 +314,7 @@ public class DhLightingEngine
// propagate the lighting in each cardinal direction, IE: -x, +x, -y, +y, -z, +z
for (EDhDirection direction : EDhDirection.CARDINAL_DIRECTIONS) // since this is an array instead of an ArrayList this advanced for-loop shouldn't cause any GC issues
for (EDhDirection direction : EDhDirection.ALL) // since this is an array instead of an ArrayList this advanced for-loop shouldn't cause any GC issues
{
lightPos.mutateOffset(direction, neighbourBlockPos);
neighbourBlockPos.mutateToChunkRelativePos(relNeighbourBlockPos);
@@ -413,7 +413,7 @@ public class DhLightingEngine
{
for (int x = 0; x < FullDataSourceV2.WIDTH; x++)
{
LongArrayList dataPoints = dataSource.get(x, z);
LongArrayList dataPoints = dataSource.getColumnAtRelPos(x, z);
if (dataPoints != null && !dataPoints.isEmpty())
{
// iterate through the data points in this column top-down
@@ -564,7 +564,7 @@ public class DhLightingEngine
// check if the adjacent position is within the bounds of this data source...
if (adjacentX >= 0 && adjacentX < FullDataSourceV2.WIDTH && adjacentZ >= 0 && adjacentZ < FullDataSourceV2.WIDTH)
{
LongArrayList adjacentDataPoints = chunk.get(adjacentX, adjacentZ);
LongArrayList adjacentDataPoints = chunk.getColumnAtRelPos(adjacentX, adjacentZ);
// ...and also check to make sure we have some data points
// (potentially transparent ones) to propagate through in the adjacent column.
if (adjacentDataPoints != null)
core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java
+3 -1
View File
@@ -147,7 +147,9 @@ public class PregenManager
}
this.pendingGenerations.put(nextSectionPos, System.currentTimeMillis());
this.fullDataSourceProvider.getAsync(nextSectionPos).thenAccept(fullDataSource -> {
this.fullDataSourceProvider.getAsync(nextSectionPos)
.thenAccept(fullDataSource ->
{
if (this.fullDataSourceProvider.isFullyGenerated(fullDataSource.columnGenerationSteps))
{
this.pendingGenerations.invalidate(fullDataSource.getPos());
core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java
+6 -12
View File
@@ -621,19 +621,13 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
LodUtil.assertTrue(this.generatorClosingFuture != null);
LOGGER.info("Awaiting world generator thread pool termination...");
try
LOGGER.info("Shutting down world generator thread pool...");
AbstractExecutorService executor = ThreadPoolUtil.getWorldGenExecutor();
if (executor != null)
{
int waitTimeInSeconds = 3;
AbstractExecutorService executor = ThreadPoolUtil.getWorldGenExecutor();
if (executor != null && !executor.awaitTermination(waitTimeInSeconds, TimeUnit.SECONDS))
{
LOGGER.warn("World generator thread pool shutdown didn't complete after [" + waitTimeInSeconds + "] seconds. Some world generator requests may still be running.");
}
}
catch (InterruptedException e)
{
LOGGER.warn("World generator thread pool shutdown interrupted! Ignoring child threads...", e);
List<Runnable> tasks = executor.shutdownNow();
LOGGER.info("World generator thread pool shutdown with [" + tasks.size() + "] incomplete tasks.");
}
core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhServerLevel.java
+2 -23
View File
@@ -2,10 +2,9 @@ package com.seibel.distanthorizons.core.level;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.logging.f3.F3Screen;
import com.seibel.distanthorizons.core.multiplayer.server.FullDataSourceRequestHandler;
import com.seibel.distanthorizons.core.multiplayer.server.ServerPlayerState;
import com.seibel.distanthorizons.core.multiplayer.server.ServerPlayerStateManager;
@@ -297,27 +296,7 @@ public abstract class AbstractDhServerLevel extends AbstractDhLevel implements I
@Override
public void addDebugMenuStringsToList(List<String> messageList)
{
// migration
boolean migrationErrored = this.serverside.fullDataFileHandler.getMigrationStoppedWithError();
if (!migrationErrored)
{
long legacyDeletionCount = this.serverside.fullDataFileHandler.getLegacyDeletionCount();
if (legacyDeletionCount > 0)
{
messageList.add(" Migrating - Deleting #: " + F3Screen.NUMBER_FORMAT.format(legacyDeletionCount));
}
long migrationCount = this.serverside.fullDataFileHandler.getTotalMigrationCount();
if (migrationCount > 0)
{
messageList.add(" Migrating - Conversion #: " + F3Screen.NUMBER_FORMAT.format(migrationCount));
}
}
else
{
messageList.add(" Migration Failed");
}
// world gen
this.serverside.fullDataFileHandler.addDebugMenuStringsToList(messageList);
this.serverside.worldGenModule.addDebugMenuStringsToList(messageList);
}
core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java
+7 -10
View File
@@ -19,22 +19,18 @@
package com.seibel.distanthorizons.core.level;
import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering;
import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D;
import com.seibel.distanthorizons.core.render.LodQuadTree;
import com.seibel.distanthorizons.core.render.RenderBufferHandler;
import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
import com.seibel.distanthorizons.core.render.renderer.LodRenderer;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper;
import com.seibel.distanthorizons.core.logging.DhLogger;
@@ -43,7 +39,7 @@ import java.io.Closeable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicReference;
public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.IDataSourceUpdateFunc<FullDataSourceV2>
public class ClientLevelModule implements Closeable, IDataSourceUpdateListenerFunc<FullDataSourceV2>
{
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
@@ -73,7 +69,7 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I
this.clientLevel = clientLevel;
this.fullDataSourceProvider = this.clientLevel.getFullDataProvider();
this.fullDataSourceProvider.dateSourceUpdateListeners.add(this);
this.fullDataSourceProvider.addDataSourceUpdateListener(this);
}
@@ -165,7 +161,8 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I
// data handling //
//===============//
public CompletableFuture<Void> updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientLevel.getFullDataProvider().updateDataSourceAsync(data); }
public CompletableFuture<Void> updateDataSourcesAsync(FullDataSourceV2 data)
{ return this.clientLevel.getFullDataProvider().updateDataSourceAsync(data); }
@Override
public void OnDataSourceUpdated(FullDataSourceV2 updatedFullDataSource)
{
@@ -199,7 +196,7 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I
}
}
this.fullDataSourceProvider.dateSourceUpdateListeners.remove(this);
this.fullDataSourceProvider.removeDataSourceUpdateListener(this);
}
core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java
+9 -26
View File
@@ -23,7 +23,7 @@ import com.google.common.cache.CacheBuilder;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.generation.RemoteWorldRetrievalQueue;
@@ -68,7 +68,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
public final ClientLevelModule clientside;
public final IClientLevelWrapper levelWrapper;
public final ISaveStructure saveStructure;
public final RemoteFullDataSourceProvider dataFileHandler;
public final RemoteFullDataSourceProvider remoteDataSourceProvider;
@CheckForNull
private final ClientNetworkState networkState;
@@ -130,12 +130,12 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
this.syncOnLoadRequestQueue = null;
}
this.dataFileHandler = new RemoteFullDataSourceProvider(this, saveStructure, fullDataSaveDirOverride, this.syncOnLoadRequestQueue);
this.worldGenModule = new WorldGenModule(this, this.dataFileHandler, () -> new WorldGenState(this, networkState));
this.remoteDataSourceProvider = new RemoteFullDataSourceProvider(this, saveStructure, fullDataSaveDirOverride, this.syncOnLoadRequestQueue);
this.worldGenModule = new WorldGenModule(this, this.remoteDataSourceProvider, () -> new WorldGenState(this, networkState));
this.clientside = new ClientLevelModule(this);
this.createAndSetSupportingRepos(this.dataFileHandler.repo.databaseFile);
this.createAndSetSupportingRepos(this.remoteDataSourceProvider.repo.databaseFile);
this.runRepoReliantSetup();
this.clientside.startRenderer();
@@ -182,7 +182,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
}
FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.levelWrapper);
FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.levelWrapper, null);
this.updateDataSourcesAsync(fullDataSource)
.whenComplete((result, e) -> fullDataSource.close());
}
@@ -283,7 +283,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
public CompletableFuture<Void> updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientside.updateDataSourcesAsync(data); }
@Override
public FullDataSourceProviderV2 getFullDataProvider() { return this.dataFileHandler; }
public FullDataSourceProviderV2 getFullDataProvider() { return this.remoteDataSourceProvider; }
@Override
public ISaveStructure getSaveStructure() { return this.saveStructure; }
@@ -321,24 +321,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
messageList.add("["+dimName+"] rendering: "+(rendering ? "yes" : "no"));
boolean migrationErrored = this.dataFileHandler.getMigrationStoppedWithError();
if (!migrationErrored)
{
long legacyDeletionCount = this.dataFileHandler.getLegacyDeletionCount();
if (legacyDeletionCount > 0)
{
messageList.add(" Migrating - Deleting #: " + legacyDeletionCount);
}
long migrationCount = this.dataFileHandler.getTotalMigrationCount();
if (migrationCount > 0)
{
messageList.add(" Migrating - Conversion #: " + migrationCount);
}
}
else
{
messageList.add(" Migration Failed");
}
this.remoteDataSourceProvider.addDebugMenuStringsToList(messageList);
// world gen
@@ -378,7 +361,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel
this.levelWrapper.setDhLevel(null);
this.clientside.close();
super.close();
this.dataFileHandler.close();
this.remoteDataSourceProvider.close();
LOGGER.info("Closed [" + DhClientLevel.class.getSimpleName() + "] for [" + this.levelWrapper + "]");
}
core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java
-1
View File
@@ -107,7 +107,6 @@ public class DhServerLevel extends AbstractDhServerLevel
{
super.close();
this.serverside.close();
LOGGER.info("Closed DHLevel for ["+this.getLevelWrapper()+"].");
}
}
core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java
+1 -1
View File
@@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.level;
import com.seibel.distanthorizons.core.api.internal.ClientApi;
import com.seibel.distanthorizons.core.api.internal.ServerApi;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider;
import com.seibel.distanthorizons.core.file.structure.ISaveStructure;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java
+1 -1
View File
@@ -279,7 +279,7 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende
{
this.level.updateBeaconBeamsForSectionPos(dataSourceDto.pos, response.payload.beaconBeams);
FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper());
FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper(), null);
entry.dataSourceConsumer.accept(fullDataSource);
}
catch (Exception e)
core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java
+2 -2
View File
@@ -344,8 +344,8 @@ public class DhSectionPos
}
return DhSectionPos.encode(getDetailLevel(pos),
getX(pos) + dir.getNormal().x,
getZ(pos) + dir.getNormal().z);
getX(pos) + dir.normal.x,
getZ(pos) + dir.normal.z);
}
@Deprecated
core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPos.java
+2 -2
View File
@@ -75,9 +75,9 @@ public class DhBlockPos implements INetworkObject
//========//
/** creates a new {@link DhBlockPos} with the given offset from the current pos. */
public DhBlockPos createOffset(EDhDirection direction) { return this.mutateOrCreateOffset(direction.getNormal().x, direction.getNormal().y, direction.getNormal().z, null); }
public DhBlockPos createOffset(EDhDirection direction) { return this.mutateOrCreateOffset(direction.normal.x, direction.normal.y, direction.normal.z, null); }
/** if not null, mutates "mutablePos" so it matches the current pos after being offset. Otherwise creates a new {@link DhBlockPos}. */
public void mutateOffset(EDhDirection direction, @NotNull DhBlockPosMutable mutablePos) { this.mutateOrCreateOffset(direction.getNormal().x, direction.getNormal().y, direction.getNormal().z, mutablePos); }
public void mutateOffset(EDhDirection direction, @NotNull DhBlockPosMutable mutablePos) { this.mutateOrCreateOffset(direction.normal.x, direction.normal.y, direction.normal.z, mutablePos); }
public DhBlockPos createOffset(int x, int y, int z) { return this.mutateOrCreateOffset(x,y,z, null); }
public void mutateOffset(int x, int y, int z, @NotNull DhBlockPosMutable mutablePos) { this.mutateOrCreateOffset(x, y, z, mutablePos); }
core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPosMutable.java
+1 -1
View File
@@ -50,7 +50,7 @@ public class DhBlockPosMutable extends DhBlockPos
//========//
/** @see DhBlockPos#createOffset(EDhDirection) */
public DhBlockPosMutable createOffset(EDhDirection direction) { return new DhBlockPosMutable(super.mutateOrCreateOffset(direction.getNormal().x, direction.getNormal().y, direction.getNormal().z, null)); }
public DhBlockPosMutable createOffset(EDhDirection direction) { return new DhBlockPosMutable(super.mutateOrCreateOffset(direction.normal.x, direction.normal.y, direction.normal.z, null)); }
/** @see DhBlockPos#createOffset(int, int, int) */
public DhBlockPosMutable createOffset(int x, int y, int z) { return new DhBlockPosMutable(this.mutateOrCreateOffset(x,y,z, null)); }
core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java
+14 -104
View File
@@ -19,14 +19,11 @@
package com.seibel.distanthorizons.core.render;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.render.CachedColumnRenderSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -36,7 +33,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler;
import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
import com.seibel.distanthorizons.core.util.KeyedLockContainer;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.PerfRecorder;
import com.seibel.distanthorizons.core.util.ThreadUtil;
@@ -63,8 +59,6 @@ import java.util.concurrent.locks.ReentrantLock;
*/
public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRenderable, AutoCloseable
{
public static final byte TREE_LOWEST_DETAIL_LEVEL = ColumnRenderSource.SECTION_SIZE_OFFSET;
private static final DhLogger LOGGER = new DhLoggerBuilder().build();
/** there should only ever be one {@link LodQuadTree} so having the thread static should be fine */
private static final ThreadPoolExecutor FULL_DATA_RETRIEVAL_QUEUE_THREAD = ThreadUtil.makeSingleThreadPool("QuadTree Full Data Retrieval Queue Populator");
@@ -87,28 +81,6 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
private ArrayList<LodRenderSection> altDebugRenderSections = new ArrayList<>();
private final ReentrantLock debugRenderSectionLock = new ReentrantLock();
/** don't let two threads load the same position at the same time */
protected final KeyedLockContainer<Long> renderLoadLockContainer = new KeyedLockContainer<>();
/**
* caching is done at the QuadTree level to prevent caching LODs for different levels.
* (Although the incorrect terrain that renders is quite entertaining). <br><br>
*
* caching the loaded positions significantly improves initial loading performance
* since the same position doesn't need to be loaded 5 times.
*/
private final Cache<Long, CachedColumnRenderSource> cachedRenderSourceByPos
= CacheBuilder.newBuilder()
// availableProcessors() : each process may need to be loading a render source
// +1 : add 1 thread count buffer to reduce the chance of accidentally unloading a render source before it's used
// *5 : each render source needs it's 4 adjacent sides, so a total of 5 render sources are needed per load
.maximumSize((Runtime.getRuntime().availableProcessors() + 1) * 5L)
// No closing logic since the CachedColumnRenderSource is in charge
// of freeing the underlying ColumnRenderSource.
// That way we don't have to worry about accidentally closing an in-use object.
.<Long, CachedColumnRenderSource>build();
/**
* Used to limit how many upload tasks are queued at once.
* If all the upload tasks are queued at once, they will start uploading nearest
@@ -122,14 +94,10 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
@Nullable
public final BeaconRenderHandler beaconRenderHandler;
// TODO should be removed once James is done testing
@Deprecated
public static final PerfRecorder FILE_PERF_RECORDER = new PerfRecorder("File");
/** the smallest numerical detail level number that can be rendered */
private byte maxRenderDetailLevel;
private byte maxLeafRenderDetailLevel;
/** the largest numerical detail level number that can be rendered */
private byte minRenderDetailLevel;
private byte minRootRenderDetailLevel;
/** used to calculate when a detail drop will occur */
private double detailDropOffDistanceUnit;
@@ -147,7 +115,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
int initialPlayerBlockX, int initialPlayerBlockZ,
FullDataSourceProviderV2 fullDataSourceProvider)
{
super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL);
super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showQuadTreeRenderStatus);
@@ -158,8 +126,6 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
GenericObjectRenderer genericObjectRenderer = this.level.getGenericRenderer();
this.beaconRenderHandler = (genericObjectRenderer != null) ? new BeaconRenderHandler(genericObjectRenderer) : null;
FILE_PERF_RECORDER.clear();
}
@@ -240,7 +206,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
long rootPos = rootPosIterator.nextLong();
if (this.getNode(rootPos) == null)
{
this.setValue(rootPos, new LodRenderSection(rootPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer));
this.setValue(rootPos, new LodRenderSection(rootPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef));
}
QuadNode<LodRenderSection> rootNode = this.getNode(rootPos);
@@ -281,7 +247,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
// create the node
if (quadNode == null && this.isSectionPosInBounds(sectionPos)) // the position bounds should only fail when at the edge of the user's render distance
{
rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer));
rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef));
quadNode = rootNode.getNode(sectionPos);
}
if (quadNode == null)
@@ -294,7 +260,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
LodRenderSection renderSection = quadNode.value;
if (renderSection == null)
{
renderSection = new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer);
renderSection = new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef);
quadNode.setValue(sectionPos, renderSection);
}
@@ -305,9 +271,8 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
// and disabling render sections //
//===============================//
//byte expectedDetailLevel = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + 3; // can be used instead of the following logic for testing
byte expectedDetailLevel = this.calculateExpectedDetailLevel(playerPos, sectionPos);
expectedDetailLevel = (byte) Math.min(expectedDetailLevel, this.minRenderDetailLevel);
expectedDetailLevel = (byte) Math.min(expectedDetailLevel, this.minRootRenderDetailLevel);
expectedDetailLevel += DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL;
if (DhSectionPos.getDetailLevel(sectionPos) > expectedDetailLevel)
@@ -555,11 +520,11 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
int detailLevel = (int) (Math.log(distance / this.detailDropOffDistanceUnit) / this.detailDropOffLogBase);
return (byte) MathUtil.clamp(this.maxRenderDetailLevel, detailLevel, Byte.MAX_VALUE - 1);
return (byte) MathUtil.clamp(this.maxLeafRenderDetailLevel, detailLevel, Byte.MAX_VALUE - 1);
}
private double getDrawDistanceFromDetail(int detail)
{
if (detail <= this.maxRenderDetailLevel)
if (detail <= this.maxLeafRenderDetailLevel)
{
return 0;
}
@@ -578,14 +543,14 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
this.detailDropOffDistanceUnit = Config.Client.Advanced.Graphics.Quality.horizontalQuality.get().distanceUnitInBlocks * LodUtil.CHUNK_WIDTH;
this.detailDropOffLogBase = Math.log(Config.Client.Advanced.Graphics.Quality.horizontalQuality.get().quadraticBase);
this.maxRenderDetailLevel = Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution.get().detailLevel;
this.maxLeafRenderDetailLevel = Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution.get().detailLevel;
// The minimum detail level is done to prevent single corner sections rendering 1 detail level lower than the others.
// If not done corners may not be flush with the other LODs, which looks bad.
byte minSectionDetailLevel = this.getDetailLevelFromDistance(this.blockRenderDistanceDiameter); // get the minimum allowed detail level
minSectionDetailLevel -= 1; // -1 so corners can't render lower than their adjacent neighbors. space
minSectionDetailLevel = (byte) Math.min(minSectionDetailLevel, this.treeRootDetailLevel); // don't allow rendering lower detail sections than what the tree contains
this.minRenderDetailLevel = (byte) Math.max(minSectionDetailLevel, this.maxRenderDetailLevel); // respect the user's selected max resolution if it is lower detail (IE they want 2x2 block, but minSectionDetailLevel is specifically for 1x1 block render resolution)
this.minRootRenderDetailLevel = (byte) Math.max(minSectionDetailLevel, this.maxLeafRenderDetailLevel); // respect the user's selected max resolution if it is lower detail (IE they want 2x2 block, but minSectionDetailLevel is specifically for 1x1 block render resolution)
}
@@ -636,17 +601,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
* This should be called whenever a world generation task is completed or if the connected server has new data to show.
*/
public void reloadPos(long pos)
{
// clear cache //
this.clearRenderCacheForPos(pos);
for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS)
{
long adjacentPos = DhSectionPos.getAdjacentPos(pos, direction);
this.clearRenderCacheForPos(adjacentPos);
}
{
// queue reloads //
// only queue each section for reloading
@@ -658,28 +613,12 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
// the adjacent locations also need to be updated to make sure lighting
// and water updates correctly, otherwise oceans may have walls
// and lights may not show up over LOD borders
for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS)
for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS)
{
long adjacentPos = DhSectionPos.getAdjacentPos(pos, direction);
this.sectionsToReload.add(adjacentPos);
}
}
private void clearRenderCacheForPos(long pos)
{
// locking is needed to prevent another thread
// from accessing the cache while it's being cleared
ReentrantLock lock = this.renderLoadLockContainer.getLockForPos(pos);
try
{
lock.lock();
this.cachedRenderSourceByPos.invalidate(pos);
}
finally
{
lock.unlock();
}
}
//=================================//
@@ -830,39 +769,10 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
LodRenderSection renderSection = quadNode.value;
if (renderSection != null)
{
// we need to wait for the render data to finish building before we can close the cache
CompletableFuture<Void> future = renderSection.getRenderDataBuildFuture();
if (future != null)
{
renderDataBuildFutures.add(future);
}
renderSection.close();
quadNode.value = null;
}
}
// close the render cache after it is done being used
LOGGER.info("waiting for ["+renderDataBuildFutures.size()+"] futures before closing render cache...");
CompletableFuture.allOf(renderDataBuildFutures.toArray(new CompletableFuture[0]))
.handle((voidObj, throwable) ->
{
// run on a separate thread so we don't lock up the main cleanup thread
// with the sleep() call
new Thread(() ->
{
// Sleep shouldn't be necessary, but James found a few cases where
// the futures incorrectly claimed they were done.
// Sleeping solved those issues.
try { Thread.sleep(5_000); } catch (InterruptedException ignore) { }
LOGGER.debug("closing render cache");
this.cachedRenderSourceByPos.invalidateAll();
}).start();
return null;
});
}
finally
{
core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java
+29 -77
View File
@@ -21,17 +21,15 @@ package com.seibel.distanthorizons.core.render;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.cache.Cache;
import com.seibel.distanthorizons.core.config.Config;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dataObjects.render.CachedColumnRenderSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder;
import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodQuadBuilder;
import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2;
import com.seibel.distanthorizons.core.level.IDhClientLevel;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
@@ -44,7 +42,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler;
import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO;
import com.seibel.distanthorizons.core.sql.repo.BeaconBeamRepo;
import com.seibel.distanthorizons.core.util.KeyedLockContainer;
import com.seibel.distanthorizons.core.util.PerfRecorder;
import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker;
import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
@@ -59,7 +56,6 @@ import java.util.*;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
/**
* A render section represents an area that could be rendered.
@@ -79,8 +75,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
@WillNotClose
private final FullDataSourceProviderV2 fullDataSourceProvider;
private final LodQuadTree quadTree;
private final KeyedLockContainer<Long> renderLoadLockContainer;
private final Cache<Long, CachedColumnRenderSource> cachedRenderSourceByPos;
private final AtomicInteger uploadTaskCountRef;
/**
@@ -134,9 +128,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
private boolean checkedIfFullDataSourceExists = false;
private boolean fullDataSourceExists = false;
@Deprecated
public final PerfRecorder filePerfRecorder = LodQuadTree.FILE_PERF_RECORDER;
//=============//
@@ -147,13 +138,10 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
long pos,
LodQuadTree quadTree,
IDhClientLevel level, FullDataSourceProviderV2 fullDataSourceProvider,
AtomicInteger uploadTaskCountRef,
Cache<Long, CachedColumnRenderSource> cachedRenderSourceByPos, KeyedLockContainer<Long> renderLoadLockContainer)
AtomicInteger uploadTaskCountRef)
{
this.pos = pos;
this.quadTree = quadTree;
this.cachedRenderSourceByPos = cachedRenderSourceByPos;
this.renderLoadLockContainer = renderLoadLockContainer;
this.level = level;
this.levelWrapper = level.getClientLevelWrapper();
this.fullDataSourceProvider = fullDataSourceProvider;
@@ -245,11 +233,11 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
{
// get the center pos data
return this.getRenderSourceForPosAsync(this.pos, null)
.thenCompose((CachedColumnRenderSource cachedRenderSource) ->
.thenCompose((ColumnRenderSource thisRenderSource) ->
{
try
{
if (cachedRenderSource == null || cachedRenderSource.columnRenderSource == null)
if (thisRenderSource == null)
{
// nothing needs to be rendered
// TODO how doesn't this cause infinite file handler loops?
@@ -257,18 +245,15 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
// setting the render buffer here
return CompletableFuture.completedFuture(null);
}
ColumnRenderSource thisRenderSource = cachedRenderSource.columnRenderSource;
boolean enableTransparency = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled;
LodQuadBuilder lodQuadBuilder = new LodQuadBuilder(enableTransparency, this.level.getClientLevelWrapper());
PerfRecorder.Timer getAdj = this.filePerfRecorder.start("getAdj");
// get the adjacent positions
// needs to be done async to prevent threads waiting on the same positions to be processed
final CompletableFuture<CachedColumnRenderSource>[] adjacentLoadFutures = new CompletableFuture[4];
final CompletableFuture<ColumnRenderSource>[] adjacentLoadFutures = new CompletableFuture[4];
if (Config.Client.Advanced.Graphics.Experimental.onlyLoadCenterLods.get())
{
@@ -289,34 +274,27 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
return CompletableFuture.allOf(adjacentLoadFutures).thenRun(() ->
{
getAdj.end();
try (CachedColumnRenderSource northRenderSource = adjacentLoadFutures[0].get();
CachedColumnRenderSource southRenderSource = adjacentLoadFutures[1].get();
CachedColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get();
CachedColumnRenderSource westRenderSource = adjacentLoadFutures[3].get())
try (ColumnRenderSource northRenderSource = adjacentLoadFutures[0].get();
ColumnRenderSource southRenderSource = adjacentLoadFutures[1].get();
ColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get();
ColumnRenderSource westRenderSource = adjacentLoadFutures[3].get())
{
ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length];
adjacentRenderSections[EDhDirection.NORTH.ordinal() - 2] = (northRenderSource != null) ? northRenderSource.columnRenderSource : null;
adjacentRenderSections[EDhDirection.SOUTH.ordinal() - 2] = (southRenderSource != null) ? southRenderSource.columnRenderSource : null;
adjacentRenderSections[EDhDirection.EAST.ordinal() - 2] = (eastRenderSource != null) ? eastRenderSource.columnRenderSource : null;
adjacentRenderSections[EDhDirection.WEST.ordinal() - 2] = (westRenderSource != null) ? westRenderSource.columnRenderSource : null;
ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.CARDINAL_COMPASS.length];
adjacentRenderSections[EDhDirection.NORTH.compassIndex] = northRenderSource;
adjacentRenderSections[EDhDirection.SOUTH.compassIndex] = southRenderSource;
adjacentRenderSections[EDhDirection.EAST.compassIndex] = eastRenderSource;
adjacentRenderSections[EDhDirection.WEST.compassIndex] = westRenderSource;
boolean[] adjIsSameDetailLevel = new boolean[EDhDirection.ADJ_DIRECTIONS.length];
adjIsSameDetailLevel[EDhDirection.NORTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.NORTH);
adjIsSameDetailLevel[EDhDirection.SOUTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.SOUTH);
adjIsSameDetailLevel[EDhDirection.EAST.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.EAST);
adjIsSameDetailLevel[EDhDirection.WEST.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.WEST);
boolean[] adjIsSameDetailLevel = new boolean[EDhDirection.CARDINAL_COMPASS.length];
adjIsSameDetailLevel[EDhDirection.NORTH.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.NORTH);
adjIsSameDetailLevel[EDhDirection.SOUTH.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.SOUTH);
adjIsSameDetailLevel[EDhDirection.EAST.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.EAST);
adjIsSameDetailLevel[EDhDirection.WEST.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.WEST);
// the render sources are only needed by this synchronous method,
// then they can be closed
PerfRecorder.Timer makeRender = this.filePerfRecorder.start("makeRender");
ColumnRenderBufferBuilder.makeLodRenderData(lodQuadBuilder, thisRenderSource, this.level, adjacentRenderSections, adjIsSameDetailLevel);
makeRender.end();
PerfRecorder.Timer upload = this.filePerfRecorder.start("upload");
this.uploadToGpuAsync(lodQuadBuilder);
upload.end();
}
catch (Exception e)
{
@@ -325,7 +303,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
finally
{
// can only be closed after the data has been processed and uploaded to the GPU
cachedRenderSource.close();
thisRenderSource.close();
}
});
}
@@ -337,7 +315,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
});
}
/** async is done so each thread can run without waiting on others */
private CompletableFuture<CachedColumnRenderSource> getRenderSourceForPosAsync(long pos, @Nullable EDhDirection direction)
private CompletableFuture<ColumnRenderSource> getRenderSourceForPosAsync(long pos, @Nullable EDhDirection direction)
{
if (direction != null)
{
@@ -346,23 +324,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
final long finalPos = pos;
ReentrantLock lock = this.renderLoadLockContainer.getLockForPos(finalPos);
try
{
// we don't want multiple threads attempting to load the same position at the same time,
// and we don't want to access the cache while invalidating it on a different thread
lock.lock();
// use the cached data if possible
CachedColumnRenderSource existingCachedRenderSource = this.cachedRenderSourceByPos.getIfPresent(finalPos);
if (existingCachedRenderSource != null)
{
existingCachedRenderSource.markInUse();
return existingCachedRenderSource.loadFuture;
}
PriorityTaskPicker.Executor executor = ThreadPoolUtil.getRenderLoadingExecutor();
if (executor == null || executor.isTerminated())
{
@@ -372,31 +335,24 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
// queue loading the render data
CompletableFuture<CachedColumnRenderSource> loadFuture = new CompletableFuture<>();
final CachedColumnRenderSource newCachedRenderSource = new CachedColumnRenderSource(loadFuture, lock, this.cachedRenderSourceByPos);
CompletableFuture<ColumnRenderSource> loadFuture = new CompletableFuture<>();
executor.execute(() ->
{
PerfRecorder.Timer getFull = this.filePerfRecorder.start("getFull");
// generate new render source
try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(finalPos))
try (FullDataSourceV2 fullDataSource =
// no direction means get the center LOD
(direction == null)
? this.fullDataSourceProvider.get(finalPos)
: this.fullDataSourceProvider.getAdjForDirection(finalPos, direction.opposite()))
{
getFull.end();
PerfRecorder.Timer transform = this.filePerfRecorder.start("transform");
newCachedRenderSource.columnRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.levelWrapper);
transform.end();
ColumnRenderSource columnRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.levelWrapper);
loadFuture.complete(columnRenderSource);
}
catch (Exception e)
{
LOGGER.error("Unexpected issue creating render data for pos: ["+DhSectionPos.toString(finalPos)+"], error: ["+e.getMessage()+"].", e);
}
finally
{
loadFuture.complete(newCachedRenderSource);
}
});
this.cachedRenderSourceByPos.put(pos, newCachedRenderSource);
return loadFuture;
}
@@ -410,10 +366,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable
LOGGER.error("Unexpected issue getting and creating render data for pos: ["+DhSectionPos.toString(pos)+"], error: ["+e.getMessage()+"].", e);
return CompletableFuture.completedFuture(null);
}
finally
{
lock.unlock();
}
}
private boolean isAdjacentPosSameDetailLevel(EDhDirection direction)
{
core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java
+10 -1
View File
@@ -220,7 +220,16 @@ public class GLProxy
return instance;
}
public EDhApiGpuUploadMethod getGpuUploadMethod() { return this.preferredUploadMethod; }
public EDhApiGpuUploadMethod getGpuUploadMethod()
{
EDhApiGpuUploadMethod uploadOverride = Config.Client.Advanced.Debugging.OpenGl.glUploadMode.get();
if (uploadOverride == EDhApiGpuUploadMethod.AUTO)
{
return this.preferredUploadMethod;
}
return uploadOverride;
}
public boolean runningOnRenderThread()
{
core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java
+4 -3
View File
@@ -27,7 +27,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.render.glObject.GLProxy;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.math.UnitBytes;
import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftGLWrapper;
import org.lwjgl.opengl.GL32;
import org.lwjgl.opengl.GL44;
@@ -187,7 +186,6 @@ public class GLBuffer implements AutoCloseable
{
LodUtil.assertNotReach("maxExpansionSize is [" + maxExpansionSize + "] but buffer size is [" + bbSize + "]!");
}
GLProxy.LOGGER.debug("Uploading buffer with ["+new UnitBytes(bbSize)+"].");
// Don't upload an empty buffer
if (bbSize == 0)
@@ -200,6 +198,8 @@ public class GLBuffer implements AutoCloseable
switch (uploadMethod)
{
case NONE:
return;
case AUTO:
LodUtil.assertNotReach("GpuUploadMethod AUTO must be resolved before call to uploadBuffer()!");
case BUFFER_STORAGE:
@@ -379,6 +379,7 @@ public class GLBuffer implements AutoCloseable
{
int id = PHANTOM_TO_BUFFER_ID.get(phantomRef);
destroyBufferIdAsync(id);
LOGGER.warn("Buffer Phantom collected, ID: ["+id+"]");
}
phantomRef = PHANTOM_REFERENCE_QUEUE.poll();
@@ -386,7 +387,7 @@ public class GLBuffer implements AutoCloseable
}
catch (Exception e)
{
LOGGER.error("Unexpected error in cleanup thread: [" + e.getMessage() + "].", e);
LOGGER.error("Unexpected error in buffer cleanup thread: [" + e.getMessage() + "].", e);
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java
+405 -27
View File
@@ -25,10 +25,13 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList;
import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.network.INetworkObject;
import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil;
import com.seibel.distanthorizons.core.sql.dto.util.VarintUtil;
import com.seibel.distanthorizons.core.util.BoolUtil;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.ListUtil;
@@ -52,6 +55,13 @@ public class FullDataSourceV2DTO
{
public static final boolean VALIDATE_INPUT_DATAPOINTS = true;
public static class DATA_FORMAT
{
public static final int V1_NO_ADJACENT_DATA = 1;
public static final int V2_LATEST = 2;
}
public long pos;
@@ -61,6 +71,10 @@ public class FullDataSourceV2DTO
public int dataChecksum;
public ByteArrayList compressedDataByteArray;
public ByteArrayList compressedNorthAdjDataByteArray;
public ByteArrayList compressedSouthAdjDataByteArray;
public ByteArrayList compressedEastAdjDataByteArray;
public ByteArrayList compressedWestAdjDataByteArray;
/** @see EDhApiWorldGenerationStep */
public ByteArrayList compressedColumnGenStepByteArray;
@@ -96,10 +110,15 @@ public class FullDataSourceV2DTO
FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding();
// populate arrays
writeDataSourceDataArrayToBlob(dataSource.dataPoints, dto.compressedDataByteArray, compressionModeEnum);
writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedDataByteArray, null, compressionModeEnum);
writeGenerationStepsToBlob(dataSource.columnGenerationSteps, dto.compressedColumnGenStepByteArray, compressionModeEnum);
writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, dto.compressedWorldCompressionModeByteArray, compressionModeEnum);
writeDataMappingToBlob(dataSource.mapping, dto.compressedMappingByteArray, compressionModeEnum);
// adjacent full data
writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedNorthAdjDataByteArray, EDhDirection.NORTH, compressionModeEnum);
writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedSouthAdjDataByteArray, EDhDirection.SOUTH, compressionModeEnum);
writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedEastAdjDataByteArray, EDhDirection.EAST, compressionModeEnum);
writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedWestAdjDataByteArray, EDhDirection.WEST, compressionModeEnum);
// populate individual variables
{
@@ -107,7 +126,7 @@ public class FullDataSourceV2DTO
// the mapping hash isn't included since it takes significantly longer to calculate and
// as of the time of this comment (2025-1-22) the checksum isn't used for anything so changing it shouldn't cause any issues
dto.dataChecksum = dataSource.hashCode();
dto.dataFormatVersion = FullDataSourceV2.DATA_FORMAT_VERSION;
dto.dataFormatVersion = DATA_FORMAT.V2_LATEST;
dto.compressionModeValue = compressionModeEnum.value;
dto.lastModifiedUnixDateTime = dataSource.lastModifiedUnixDateTime;
dto.createdUnixDateTime = dataSource.createdUnixDateTime;
@@ -123,7 +142,7 @@ public class FullDataSourceV2DTO
public static FullDataSourceV2DTO CreateEmptyDataSourceForDecoding() { return new FullDataSourceV2DTO(); }
private FullDataSourceV2DTO()
{
super(ARRAY_LIST_POOL, 4, 0, 0);
super(ARRAY_LIST_POOL, 8, 0, 0);
// Expected sizes here are 0 since we don't know how big these arrays need to be,
// they depend on compression settings and world complexity.
@@ -131,6 +150,11 @@ public class FullDataSourceV2DTO
this.compressedColumnGenStepByteArray = this.pooledArraysCheckout.getByteArray(1, 0);
this.compressedWorldCompressionModeByteArray = this.pooledArraysCheckout.getByteArray(2, 0);
this.compressedMappingByteArray = this.pooledArraysCheckout.getByteArray(3, 0);
this.compressedNorthAdjDataByteArray = this.pooledArraysCheckout.getByteArray(4, 0);
this.compressedSouthAdjDataByteArray = this.pooledArraysCheckout.getByteArray(5, 0);
this.compressedEastAdjDataByteArray = this.pooledArraysCheckout.getByteArray(6, 0);
this.compressedWestAdjDataByteArray = this.pooledArraysCheckout.getByteArray(7, 0);
}
@@ -139,12 +163,12 @@ public class FullDataSourceV2DTO
// data source population //
//========================//
public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException
public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper, EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException
{
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos);
try
{
this.internalPopulateDataSource(dataSource, levelWrapper, false);
this.populateDataSource(dataSource, levelWrapper, direction, false);
}
catch (Exception e)
{
@@ -155,37 +179,97 @@ public class FullDataSourceV2DTO
return dataSource;
}
/**
* May be missing one or more data fields. <br>
* Designed to be used without access to Minecraft.
*/
public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException
{ return this.createUnitTestDataSource(null); }
/**
* May be missing one or more data fields. <br>
* Designed to be used without access to Minecraft or any supporting objects.
* Designed to be used without access to Minecraft.
*/
public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException
{ return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, true); }
public FullDataSourceV2 createUnitTestDataSource(EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException
{ return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); }
private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException
private FullDataSourceV2 populateDataSource(
FullDataSourceV2 dataSource, ILevelWrapper levelWrapper,
@Nullable EDhDirection direction,
boolean unitTest) throws IOException, InterruptedException, DataCorruptedException
{
if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion)
// format validation //
if (this.dataFormatVersion != DATA_FORMAT.V1_NO_ADJACENT_DATA
&& this.dataFormatVersion != DATA_FORMAT.V2_LATEST)
{
throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"].");
throw new IllegalStateException("Data source population only supports formats: ["+DATA_FORMAT.V1_NO_ADJACENT_DATA +","+DATA_FORMAT.V2_LATEST +"], data format found: ["+this.dataFormatVersion+"].");
}
if (direction != null
&& this.dataFormatVersion == DATA_FORMAT.V1_NO_ADJACENT_DATA)
{
throw new IllegalStateException("Data format ["+this.dataFormatVersion+"] doesn't support adjacent data. Automatic conversion must be done.");
}
// compression //
EDhApiDataCompressionMode compressionModeEnum;
try
{
compressionModeEnum = this.getCompressionMode();
compressionModeEnum = EDhApiDataCompressionMode.getFromValue(this.compressionModeValue);
}
catch (IllegalArgumentException e)
{
// may happen if ZStd was used (which was added and removed during the nightly builds)
// or if the compressor value is changed to an invalid option
// may happen if the compressor value was changed to an invalid option
throw new DataCorruptedException(e);
}
readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum);
readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum);
readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum);
// data //
// clear any old data so we can start fresh
for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++)
{
@NotNull LongArrayList array = dataSource.dataPoints[i];
array.clear();
array.add(FullDataPointUtil.EMPTY_DATA_POINT);
}
if (direction == null)
{
readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum);
readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum);
if (this.dataFormatVersion == 1)
{
readBlobToDataSourceDataArrayV1(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum);
}
else
{
// doesn't include adjacent (ie edge) data
readBlobToDataSourceDataArrayV2(this.compressedDataByteArray, dataSource.dataPoints, null, compressionModeEnum);
readBlobToDataSourceDataArrayV2(this.compressedNorthAdjDataByteArray, dataSource.dataPoints, EDhDirection.NORTH, compressionModeEnum);
readBlobToDataSourceDataArrayV2(this.compressedSouthAdjDataByteArray, dataSource.dataPoints, EDhDirection.SOUTH, compressionModeEnum);
readBlobToDataSourceDataArrayV2(this.compressedEastAdjDataByteArray, dataSource.dataPoints, EDhDirection.EAST, compressionModeEnum);
readBlobToDataSourceDataArrayV2(this.compressedWestAdjDataByteArray, dataSource.dataPoints, EDhDirection.WEST, compressionModeEnum);
}
}
else
{
// adjacent data is stored in the same byte array
// as the normal data,
// this is done so data sources down-stream
// can all be handled identically regardless of
// whether they're a full or partial data source
readBlobToDataSourceDataArrayV2(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum);
}
// mapping //
dataSource.mapping.clear(dataSource.getPos());
// should only be null when used in a unit test
@@ -205,6 +289,10 @@ public class FullDataSourceV2DTO
}
}
// individual properties //
dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime;
dataSource.createdUnixDateTime = this.createdUnixDateTime;
@@ -230,7 +318,9 @@ public class FullDataSourceV2DTO
// (de)serializing //
//=================//
private static void writeDataSourceDataArrayToBlob(LongArrayList[] inputDataArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
public static void writeDataSourceDataArrayToBlobV1(
LongArrayList[] inputDataArray, ByteArrayList outputByteArray,
EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
// write the outputs to a stream to prep for writing to the database
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -265,7 +355,9 @@ public class FullDataSourceV2DTO
outputByteArray.addElements(0, byteArrayOutputStream.toByteArray());
}
}
private static void readBlobToDataSourceDataArray(ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
private static void readBlobToDataSourceDataArrayV1(
ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray,
EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException
{
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements());
try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum))
@@ -298,6 +390,300 @@ public class FullDataSourceV2DTO
}
}
private static void writeDataSourceDataArrayToBlobV2(
LongArrayList[] inputDataArray, ByteArrayList outputByteArray,
@Nullable EDhDirection direction, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
int minX, maxX, minZ, maxZ;
if (direction != null)
{
long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction);
minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos);
maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos);
minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos);
maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos);
}
else
{
// skip the border data so we don't duplicate the adjacent data
minX = 1;
maxX = FullDataSourceV2.WIDTH-1;
minZ = 1;
maxZ = FullDataSourceV2.WIDTH-1;
}
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
try (DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum))
{
// this method would be simpler if we allocated a bunch of temporary arrays,
// but we're trying to avoid garbage.
// 1. column lengths
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x,z);
LongArrayList col = inputDataArray[index];
int size = (col != null) ? col.size() : 0;
VarintUtil.writeVarint(compressedOut, size);
}
}
// 2. column ids, with "is lit" and "is discontinuous" bits
int previousBottomY = 0;
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = inputDataArray[index];
int size = col != null ? col.size() : 0;
for (int y = 0; y < size; y++)
{
long data = col.getLong(y);
int id = FullDataPointUtil.getId(data);
int height = FullDataPointUtil.getHeight(data);
int bottomY = FullDataPointUtil.getBottomY(data);
boolean hasLight = (FullDataPointUtil.getBlockLight(data) | FullDataPointUtil.getSkyLight(data)) != LodUtil.MIN_MC_LIGHT;
// all datapoints are contiguous, with no gaps
// so having both height and bottomY is redundant. We could store the prediction
// in an array, but it's much cheaper to just recompute it later.
int expectedBottomY = previousBottomY - height;
boolean hasDiscontinuity = bottomY != expectedBottomY;
previousBottomY = bottomY;
VarintUtil.writeVarint(compressedOut, (id << 2) | (hasLight ? 2 : 0) | (hasDiscontinuity ? 1 : 0));
}
}
}
// 3. heights
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = inputDataArray[index];
int size = (col != null) ? col.size() : 0;
for (int y = 0; y < size; y++)
{
long data = col.getLong(y);
VarintUtil.writeVarint(compressedOut, FullDataPointUtil.getHeight(data));
}
}
}
// 4. bottomY (only the mis-predicted ones)
previousBottomY = 0;
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = inputDataArray[index];
int size = (col != null) ? col.size() : 0;
for (int y = 0; y < size; y++)
{
long data = col.getLong(y);
int height = FullDataPointUtil.getHeight(data);
int bottomY = FullDataPointUtil.getBottomY(data);
int expectedBottomY = previousBottomY - height;
if (bottomY != expectedBottomY)
{
VarintUtil.writeVarint(compressedOut, VarintUtil.zigzagEncode(bottomY - expectedBottomY));
}
previousBottomY = bottomY;
}
}
}
// 5. packed Light (only lit sections)
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = inputDataArray[index];
int size = (col != null) ? col.size() : 0;
for (int y = 0; y < size; y++)
{
long data = col.getLong(y);
int blockLight = FullDataPointUtil.getBlockLight(data);
int skyLight = FullDataPointUtil.getSkyLight(data);
byte packedLight = (byte) ((blockLight << 4) | skyLight);
if (packedLight != 0)
{
compressedOut.writeByte(packedLight);
}
}
}
}
compressedOut.flush();
byteArrayOutputStream.close();
outputByteArray.addElements(0, byteArrayOutputStream.toByteArray());
}
}
private static void readBlobToDataSourceDataArrayV2(
ByteArrayList inputCompressedDataByteArray,
LongArrayList[] outputDataLongArray,
@Nullable EDhDirection direction, EDhApiDataCompressionMode compressionModeEnum)
throws IOException, DataCorruptedException
{
int minX, maxX, minZ, maxZ;
if (direction != null)
{
long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction);
minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos);
maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos);
minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos);
maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos);
}
else
{
// skip the border data so we don't duplicate the adjacent data
minX = 1;
maxX = FullDataSourceV2.WIDTH-1;
minZ = 1;
maxZ = FullDataSourceV2.WIDTH-1;
}
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements());
try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum))
{
// 1. column counts, preallocate
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
int count = VarintUtil.readVarint(compressedIn);
ListUtil.clearAndSetSize(outputDataLongArray[index], count);
}
}
// 2. ids and flags for min_y and light
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = outputDataLongArray[index];
for (int i = 0; i < col.size(); i++)
{
int encodedId = VarintUtil.readVarint(compressedIn);
col.set(i, FullDataPointUtil.encode(encodedId >> 2, 1, encodedId & 1, (byte) (encodedId & 2), (byte) 0));
}
}
}
// 3. height
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = outputDataLongArray[index];
for (int i = 0; i < col.size(); i++)
{
int height = VarintUtil.readVarint(compressedIn);
long data = col.getLong(i);
col.set(i, FullDataPointUtil.setHeight(data, height));
}
}
}
// 4. bottomY
int previousBottomY = 0;
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = outputDataLongArray[index];
for (int i = 0; i < col.size(); i++)
{
long data = col.getLong(i);
int error = 0;
if (FullDataPointUtil.getBottomY(data) != 0)
{
error = VarintUtil.zigzagDecode(VarintUtil.readVarint(compressedIn));
}
int bottomY = previousBottomY - FullDataPointUtil.getHeight(data) + error;
col.set(i, FullDataPointUtil.setBottomY(data, bottomY));
previousBottomY = bottomY;
}
}
}
// 5. lights
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = outputDataLongArray[index];
for (int i = 0; i < col.size(); i++)
{
long data = col.getLong(i);
boolean hasLight = FullDataPointUtil.getBlockLight(data) != 0;
byte skyLight = 0;
byte blockLight = 0;
if (hasLight)
{
byte packedLight = compressedIn.readByte();
skyLight = (byte) (packedLight & 0xF);
blockLight = (byte) (packedLight >> 4);
}
col.set(i, FullDataPointUtil.setSkyLight(
FullDataPointUtil.setBlockLight(data, blockLight),
skyLight));
}
}
}
if (FullDataPointUtil.RUN_VALIDATION)
{
// These points all bypassed validation because of using setters.
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList col = outputDataLongArray[index];
for (int i = 0; i < col.size(); i++)
{
FullDataPointUtil.validateDatapoint(col.getLong(i));
}
}
}
}
}
catch (EOFException e)
{
throw new DataCorruptedException(e);
}
}
private static void writeGenerationStepsToBlob(ByteArrayList inputColumnGenStepByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException
{
@@ -443,14 +829,6 @@ public class FullDataSourceV2DTO
//================//
// helper methods //
//================//
public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); }
//===========//
// overrides //
//===========//
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java
+108
View File
@@ -0,0 +1,108 @@
package com.seibel.distanthorizons.core.sql.dto.util;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.enums.EDhDirection;
/**
* Handles encoding/decoding of min/max X/Z relative {@link FullDataSourceV2#dataPoints}
* positions. <br>
* Needed so we can keep the same format between complete data sources
* and incomplete adjacent-only data sources.
*/
public class FullDataMinMaxPosUtil
{
private static final int ADJ_POS_MASK = (int) Math.pow(2, Short.SIZE) - 1;
private static final int MIN_X_OFFSET = 0;
private static final int MAX_X_OFFSET = Short.SIZE;
private static final int MIN_Z_OFFSET = Short.SIZE * 2;
private static final int MAX_Z_OFFSET = Short.SIZE * 3;
/**
* Encodes min/max X/Z relative {@link FullDataSourceV2#dataPoints}
* positions. <br>
* Needed so we can keep the same format between complete data sources
* and incomplete adjacent-only data sources.
*/
public static long getEncodedMinMaxPos(EDhDirection direction)
{
// 4 shorts can fit in a long, and we won't need anything longer than 64 anyway
short minX;
short maxX;
short minZ;
short maxZ;
switch (direction)
{
case NORTH:
// one row closest to the negative Z axis
minX = 0;
maxX = FullDataSourceV2.WIDTH;
minZ = 0;
maxZ = 1;
break;
case SOUTH:
// one row closest to the positive Z axis
minX = 0;
maxX = FullDataSourceV2.WIDTH;
minZ = FullDataSourceV2.WIDTH - 1;
maxZ = FullDataSourceV2.WIDTH;
break;
case EAST:
// one row closest to the positive X axis
minX = FullDataSourceV2.WIDTH - 1;
maxX = FullDataSourceV2.WIDTH;
minZ = 0;
maxZ = FullDataSourceV2.WIDTH;
break;
case WEST:
// one row closest to the Negative X axis
minX = 0;
maxX = 1;
minZ = 0;
maxZ = FullDataSourceV2.WIDTH;
break;
default:
throw new IllegalArgumentException("Unsupported direction [" + direction + "].");
}
return encodeAdjMinMaxPos(
minX, maxX,
minZ, maxZ);
}
public static long encodeAdjMinMaxPos(
short minX, short maxX,
short minZ, short maxZ
)
{
long data = 0L;
data |= (long) minX << MIN_X_OFFSET;
data |= (long) maxX << MAX_X_OFFSET;
data |= (long) minZ << MIN_Z_OFFSET;
data |= (long) maxZ << MAX_Z_OFFSET;
return data;
}
public static int getAdjMinX(long encodedMinMaxPos)
{ return (int) ((encodedMinMaxPos >> MIN_X_OFFSET) & ADJ_POS_MASK); }
public static int getAdjMaxX(long encodedMinMaxPos)
{ return (int) ((encodedMinMaxPos >> MAX_X_OFFSET) & ADJ_POS_MASK); }
public static int getAdjMinZ(long encodedMinMaxPos)
{ return (int) ((encodedMinMaxPos >> MIN_Z_OFFSET) & ADJ_POS_MASK); }
public static int getAdjMaxZ(long encodedMinMaxPos)
{ return (int) ((encodedMinMaxPos >> MAX_Z_OFFSET) & ADJ_POS_MASK); }
}
core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/VarintUtil.java
+69
View File
@@ -0,0 +1,69 @@
package com.seibel.distanthorizons.core.sql.dto.util;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import java.io.IOException;
public class VarintUtil
{
/**
* zigzagEncode maps 0=>0, -1=>1, 1=>2, -2=>3, 3=>4, etc.
* this helps encode small magnitude signed numbers as small varints.
* https://lemire.me/blog/2022/11/25/making-all-your-integers-positive-with-zigzag-encoding/
*/
public static int zigzagEncode(int n)
{
// if n is (byte)-1, this results in:
// 0b1111_1110 ^ 0b1111_1111 == 0b0000_0001
return (n << 1) ^ (n >> 31);
}
public static int zigzagDecode(int n)
{ return (n >>> 1) ^ -(n & 1); }
/**
* @param value should be a zigzag encoded value
* created via {@link VarintUtil#zigzagEncode(int)}
*/
public static void writeVarint(DhDataOutputStream out, int value) throws IOException
{
if (value < 0)
{
throw new IllegalArgumentException("varint given ["+value+"], varint only accepts positive values.");
}
while (value >= 128)
{
out.writeByte(value | 128);
value >>>= 7; // 128 = 2^7
}
out.writeByte(value);
}
public static int readVarint(DhDataInputStream in) throws IOException
{
int value = 0;
int shift = 0;
byte b;
do
{
if (shift >= 32)
{
throw new IOException("invalid varint");
}
b = in.readByte();
value |= (b & 127) << shift;
shift += 7;
}
while ((b & 128) != 0);
return value;
}
}
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java
+2 -5
View File
@@ -369,7 +369,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
if (DbConnectionClosedException.isClosedException(e))
{
throw new DbConnectionClosedException(e);
return new ArrayList<>();
}
else
{
@@ -716,10 +716,7 @@ public abstract class AbstractDhRepo<TKey, TDTO extends IBaseDTO<TKey>> implemen
/**
* should NOT start with WHERE
* Example: TODO
*/
/** should not start with WHERE */
protected abstract String CreateParameterizedWhereString();
protected void setPreparedStatementWhereClause(PreparedStatement statement, TKey key) throws SQLException { this.setPreparedStatementWhereClause(statement, 1, key); }
core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java
+181 -9
View File
@@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.sql.repo;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.logging.DhLogger;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
@@ -31,7 +32,6 @@ import com.seibel.distanthorizons.core.util.ListUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import com.seibel.distanthorizons.core.logging.DhLogger;
import org.jetbrains.annotations.Nullable;
import java.io.*;
@@ -84,6 +84,9 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
@Override @Nullable
public FullDataSourceV2DTO convertResultSetToDto(ResultSet resultSet) throws ClassCastException, IOException, SQLException
{ return this.convertResultSetToDto(resultSet, true); }
public FullDataSourceV2DTO convertResultSetToDto(ResultSet resultSet, boolean includeAdjacent) throws ClassCastException, IOException, SQLException
{
//======================//
// get statement values //
@@ -123,6 +126,15 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
dto.compressedWorldCompressionModeByteArray = putAllBytes(resultSet.getBinaryStream("ColumnWorldCompressionMode"), dto.compressedWorldCompressionModeByteArray);
dto.compressedMappingByteArray = putAllBytes(resultSet.getBinaryStream("Mapping"), dto.compressedMappingByteArray);
// adjacent full data
if (includeAdjacent)
{
dto.compressedNorthAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("NorthAdjData"), dto.compressedNorthAdjDataByteArray);
dto.compressedSouthAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("SouthAdjData"), dto.compressedSouthAdjDataByteArray);
dto.compressedEastAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("EastAdjData"), dto.compressedEastAdjDataByteArray);
dto.compressedWestAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("WestAdjData"), dto.compressedWestAdjDataByteArray);
}
// set individual variables
{
dto.pos = pos;
@@ -138,11 +150,68 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
return dto;
}
@Nullable
public FullDataSourceV2DTO convertResultSetToAdjDto(long pos, EDhDirection direction, ResultSet resultSet) throws ClassCastException, IOException, SQLException
{
//======================//
// get statement values //
//======================//
int minY = resultSet.getInt("MinY");
int dataChecksum = resultSet.getInt("DataChecksum");
byte dataFormatVersion = resultSet.getByte("DataFormatVersion");
byte compressionModeValue = resultSet.getByte("CompressionMode");
// while these values can be null in the DB, null would just equate to false
boolean applyToParent = (resultSet.getInt("ApplyToParent")) == 1;
boolean applyToChildren = (resultSet.getInt("ApplyToChildren")) == 1;
long lastModifiedUnixDateTime = resultSet.getLong("LastModifiedUnixDateTime");
long createdUnixDateTime = resultSet.getLong("CreatedUnixDateTime");
//===================//
// set DTO variables //
//===================//
FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding();
// dto.compressedNorthAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("NorthAdjData"), dto.compressedNorthAdjDataByteArray);
// set pooled arrays
dto.compressedDataByteArray = putAllBytes(resultSet.getBinaryStream("AdjData"), dto.compressedDataByteArray);
dto.compressedColumnGenStepByteArray = putAllBytes(resultSet.getBinaryStream("ColumnGenerationStep"), dto.compressedColumnGenStepByteArray);
dto.compressedWorldCompressionModeByteArray = putAllBytes(resultSet.getBinaryStream("ColumnWorldCompressionMode"), dto.compressedWorldCompressionModeByteArray);
dto.compressedMappingByteArray = putAllBytes(resultSet.getBinaryStream("Mapping"), dto.compressedMappingByteArray);
// adjacent full data
//dto.compressedNorthAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("NorthAdjData"), dto.compressedNorthAdjDataByteArray);
//dto.compressedSouthAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("SouthAdjData"), dto.compressedSouthAdjDataByteArray);
//dto.compressedEastAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("EastAdjData"), dto.compressedEastAdjDataByteArray);
//dto.compressedWestAdjDataByteArray = putAllBytes(resultSet.getBinaryStream("WestAdjData"), dto.compressedWestAdjDataByteArray);
// set individual variables
{
dto.pos = pos;
dto.dataChecksum = dataChecksum;
dto.dataFormatVersion = dataFormatVersion;
dto.compressionModeValue = compressionModeValue;
dto.lastModifiedUnixDateTime = lastModifiedUnixDateTime;
dto.createdUnixDateTime = createdUnixDateTime;
dto.applyToParent = applyToParent;
dto.applyToChildren = applyToChildren;
dto.levelMinY = minY;
}
return dto;
}
private final String insertSqlTemplate =
"INSERT INTO "+this.getTableName() + " (\n" +
" DetailLevel, PosX, PosZ, \n" +
" MinY, DataChecksum, \n" +
" Data, ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" +
" NorthAdjData, SouthAdjData, EastAdjData, WestAdjData, \n" +
" DataFormatVersion, CompressionMode, ApplyToParent, ApplyToChildren, \n" +
" LastModifiedUnixDateTime, CreatedUnixDateTime) \n" +
"VALUES( \n" +
@@ -150,6 +219,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
" ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ?, ?, ?, \n" +
" ?, ? \n" +
");";
@Override
@@ -174,6 +244,11 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedColumnGenStepByteArray.elements()), dto.compressedColumnGenStepByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedWorldCompressionModeByteArray.elements()), dto.compressedWorldCompressionModeByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedMappingByteArray.elements()), dto.compressedMappingByteArray.size());
// adjacent full data
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedNorthAdjDataByteArray.elements()), dto.compressedNorthAdjDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedSouthAdjDataByteArray.elements()), dto.compressedSouthAdjDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedEastAdjDataByteArray.elements()), dto.compressedEastAdjDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedWestAdjDataByteArray.elements()), dto.compressedWestAdjDataByteArray.size());
statement.setByte(i++, dto.dataFormatVersion);
statement.setByte(i++, dto.compressionModeValue);
@@ -204,6 +279,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
" ,ColumnGenerationStep = ? \n" +
" ,ColumnWorldCompressionMode = ? \n" +
" ,Mapping = ? \n" +
" ,NorthAdjData = ?, SouthAdjData = ?, EastAdjData = ?, WestAdjData = ? \n" +
" ,DataFormatVersion = ? \n" +
" ,CompressionMode = ? \n" +
@@ -234,6 +310,12 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedColumnGenStepByteArray.elements()), dto.compressedColumnGenStepByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedWorldCompressionModeByteArray.elements()), dto.compressedWorldCompressionModeByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedMappingByteArray.elements()), dto.compressedMappingByteArray.size());
// adjacent full data
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedNorthAdjDataByteArray.elements()), dto.compressedNorthAdjDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedSouthAdjDataByteArray.elements()), dto.compressedSouthAdjDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedEastAdjDataByteArray.elements()), dto.compressedEastAdjDataByteArray.size());
statement.setBinaryStream(i++, new ByteArrayInputStream(dto.compressedWestAdjDataByteArray.elements()), dto.compressedWestAdjDataByteArray.size());
statement.setByte(i++, dto.dataFormatVersion);
statement.setByte(i++, dto.compressionModeValue);
@@ -258,6 +340,93 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
//=================//
// partial selects //
//=================//
private final String getAdjForDirectionSqlTemplate =
"SELECT \n" +
" MinY, DataChecksum, \n" +
" ColumnGenerationStep, ColumnWorldCompressionMode, Mapping, \n" +
" DataFormatVersion, CompressionMode, ApplyToParent, ApplyToChildren, \n" +
" LastModifiedUnixDateTime, CreatedUnixDateTime, \n" +
" DIRECTION_ENUM as AdjData \n" +
"FROM "+this.getTableName() + "\n" +
" WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?; \n";
private final String getAdjForNorthDirTemplate = this.getAdjForDirectionSqlTemplate.replace("DIRECTION_ENUM", "NorthAdjData");
private final String getAdjForSouthDirTemplate = this.getAdjForDirectionSqlTemplate.replace("DIRECTION_ENUM", "SouthAdjData");
private final String getAdjForEastDirTemplate = this.getAdjForDirectionSqlTemplate.replace("DIRECTION_ENUM", "EastAdjData");
private final String getAdjForWestDirTemplate = this.getAdjForDirectionSqlTemplate.replace("DIRECTION_ENUM", "WestAdjData");
public FullDataSourceV2DTO getAdjByPosAndDirection(long pos, EDhDirection direction)
{
// parameters don't work in the select, doing so causes
// JDBC to return the wrong binary data,
// so we need to hard code the direction enum
String sql;
switch (direction)
{
case NORTH:
sql = this.getAdjForNorthDirTemplate;
break;
case SOUTH:
sql = this.getAdjForSouthDirTemplate;
break;
case EAST:
sql = this.getAdjForEastDirTemplate;
break;
case WEST:
sql = this.getAdjForWestDirTemplate;
break;
default:
throw new IllegalArgumentException();
}
try(PreparedStatement statement = this.createPreparedStatement(sql))
{
if (statement == null)
{
return null;
}
int i = 1;
statement.setInt(i++, DhSectionPos.getDetailLevel(pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL);
statement.setInt(i++, DhSectionPos.getX(pos));
statement.setInt(i++, DhSectionPos.getZ(pos));
try(ResultSet resultSet = this.query(statement))
{
if (resultSet != null
&& resultSet.next())
{
return this.convertResultSetToAdjDto(pos, direction, resultSet);
}
else
{
return null;
}
}
}
catch (SQLException | IOException e)
{
if (e instanceof SQLException
&& DbConnectionClosedException.isClosedException((SQLException)e))
{
//LOGGER.warn("Attempted to get ["+this.dtoClass.getSimpleName()+"] with primary key ["+primaryKey+"] on closed repo ["+this.connectionString+"].");
}
else
{
LOGGER.warn("Unexpected issue deserializing DTO ["+this.dtoClass.getSimpleName()+"] with pos ["+DhSectionPos.toString(pos)+"] and direction ["+direction+"]. Error: ["+e.getMessage()+"].", e);
}
return null;
}
}
//=========//
// updates //
//=========//
@@ -633,11 +802,11 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
//===============//
// helper method //
//===============//
//================//
// helper methods //
//================//
private static ByteArrayList putAllBytes(InputStream inputStream, @Nullable ByteArrayList existingArrayList) throws IOException
private static ByteArrayList putAllBytes(@Nullable InputStream inputStream, @Nullable ByteArrayList existingArrayList) throws IOException
{
if (existingArrayList == null)
{
@@ -651,11 +820,14 @@ public class FullDataSourceV2Repo extends AbstractDhRepo<Long, FullDataSourceV2D
try
{
int nextByte = inputStream.read();
while (nextByte != -1)
if (inputStream != null)
{
existingArrayList.add((byte) nextByte);
nextByte = inputStream.read();
int nextByte = inputStream.read();
while (nextByte != -1)
{
existingArrayList.add((byte) nextByte);
nextByte = inputStream.read();
}
}
}
catch (EOFException ignore) { /* shouldn't happen, but just in case */ }
core/src/main/java/com/seibel/distanthorizons/core/util/PerfRecorder.java
+8 -1
View File
@@ -175,7 +175,14 @@ public class PerfRecorder
long endTime = System.nanoTime();
long totalNano = endTime - this.startTime;
LongAdder nsAdder = PerfRecorder.this.nanoPerId.computeIfAbsent(this.id, (String id) -> new LongAdder());
LongAdder nsAdder = PerfRecorder.this.nanoPerId.get(this.id);
if (nsAdder != null)
{
nsAdder.add(totalNano);
return;
}
nsAdder = PerfRecorder.this.nanoPerId.computeIfAbsent(this.id, (String id) -> new LongAdder());
nsAdder.add(totalNano);
}
}
core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java
+5 -4
View File
@@ -24,6 +24,7 @@ import com.seibel.distanthorizons.core.level.AbstractDhLevel;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView;
import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.coreapi.ModInfo;
import com.seibel.distanthorizons.core.logging.DhLogger;
@@ -202,9 +203,9 @@ public class RenderDataPointUtil
return dataPoint & ~(HEIGHT_SHIFTED_MASK | DEPTH_SHIFTED_MASK) | height | depth;
}
/** AKA the ending/top/highest Y value above {@link AbstractDhLevel#getMinY()} */
/** AKA the ending/top/highest Y value above {@link ILevelWrapper#getMinHeight()} ()} */
public static short getYMax(long dataPoint) { return (short) ((dataPoint >>> HEIGHT_SHIFT) & HEIGHT_MASK); }
/** AKA the starting/bottom/lowest Y value above {@link AbstractDhLevel#getMinY()} */
/** AKA the starting/bottom/lowest Y value above {@link ILevelWrapper#getMinHeight()} */
public static short getYMin(long dataPoint) { return (short) ((dataPoint >>> DEPTH_SHIFT) & DEPTH_MASK); }
public static long setYMin(long dataPoint, int depth) { return (long) ((dataPoint & ~(DEPTH_MASK << DEPTH_SHIFT)) | (depth & DEPTH_MASK) << DEPTH_SHIFT); }
@@ -219,7 +220,7 @@ public class RenderDataPointUtil
public static byte getBlockMaterialId(long dataPoint) { return (byte) ((dataPoint >>> IRIS_BLOCK_MATERIAL_ID_SHIFT) & IRIS_BLOCK_MATERIAL_ID_MASK); }
public static boolean isVoid(long dataPoint) { return (((dataPoint >>> DEPTH_SHIFT) & HEIGHT_DEPTH_MASK) == 0); }
public static boolean hasZeroHeight(long dataPoint) { return (((dataPoint >>> DEPTH_SHIFT) & HEIGHT_DEPTH_MASK) == 0); }
public static boolean doesDataPointExist(long dataPoint) { return dataPoint != EMPTY_DATA; }
@@ -240,7 +241,7 @@ public class RenderDataPointUtil
{
return "null";
}
else if (isVoid(dataPoint))
else if (hasZeroHeight(dataPoint))
{
return "void";
}
core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java
+1 -1
View File
@@ -59,7 +59,7 @@ public class QuadTree<T>
*/
public final byte treeLeafDetailLevel;
private final int diameterInBlocks; // diameterInBlocks
private final int diameterInBlocks;
/** contain the actual data in the quad tree structure */
private final MovableGridRingList<QuadNode<T>> topRingList;
core/src/main/java/com/seibel/distanthorizons/core/util/threading/PriorityTaskPicker.java
-4
View File
@@ -98,10 +98,6 @@ public class PriorityTaskPicker
// Clear this executor's tasks since we no longer expect anything to execute.
executor.taskQueue.clear();
}
else
{
throw e;
}
}
}
}
core/src/main/java/com/seibel/distanthorizons/core/world/AbstractDhServerWorld.java
+20 -3
View File
@@ -10,8 +10,10 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
public abstract class AbstractDhServerWorld<TDhServerLevel extends AbstractDhServerLevel> extends AbstractDhWorld implements IDhServerWorld
@@ -134,10 +136,9 @@ public abstract class AbstractDhServerWorld<TDhServerLevel extends AbstractDhSer
@Override
public void close()
{
ArrayList<CompletableFuture<Void>> closeFutures = new ArrayList<>();
for (TDhServerLevel level : this.dhLevelByLevelWrapper.values())
{
LOGGER.info("Unloading level [" + level.getLevelWrapper().getDhIdentifier() + "].");
// level wrapper shouldn't be null, but just in case
IServerLevelWrapper serverLevelWrapper = level.getServerLevelWrapper();
if (serverLevelWrapper != null)
@@ -145,7 +146,23 @@ public abstract class AbstractDhServerWorld<TDhServerLevel extends AbstractDhSer
serverLevelWrapper.onUnload();
}
level.close();
// close levels asynchronously to speed up
// shutdown on servers with a lot of levels
CompletableFuture<Void> closeFuture = new CompletableFuture<>();
Thread closeThread = new Thread(() ->
{
level.close();
closeFuture.complete(null);
}, "level shutdown");
closeThread.start();
closeFutures.add(closeFuture);
}
// wait for all the levels to finish closing
for (CompletableFuture<Void> future : closeFutures)
{
future.join();
}
this.dhLevelByLevelWrapper.clear();
core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java
+21 -3
View File
@@ -28,9 +28,11 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
public class DhClientServerWorld extends AbstractDhServerWorld<DhClientServerLevel> implements IDhClientWorld
@@ -143,13 +145,13 @@ public class DhClientServerWorld extends AbstractDhServerWorld<DhClientServerLev
@Override
public synchronized void close()
{
ArrayList<CompletableFuture<Void>> closeFutures = new ArrayList<>();
synchronized (this.dhLevels)
{
// close each level
for (DhClientServerLevel level : this.dhLevels)
{
LOGGER.info("Unloading level [" + level.getServerLevelWrapper().getDhIdentifier() + "].");
// level wrapper shouldn't be null, but just in case
IServerLevelWrapper serverLevelWrapper = level.getServerLevelWrapper();
if (serverLevelWrapper != null)
@@ -157,10 +159,26 @@ public class DhClientServerWorld extends AbstractDhServerWorld<DhClientServerLev
serverLevelWrapper.onUnload();
}
level.close();
// close levels asynchronously to speed up
// shutdown on servers with a lot of levels
CompletableFuture<Void> closeFuture = new CompletableFuture<>();
Thread closeThread = new Thread(() ->
{
level.close();
closeFuture.complete(null);
}, "level shutdown");
closeThread.start();
closeFutures.add(closeFuture);
}
}
// wait for all the levels to finish closing
for (CompletableFuture<Void> future : closeFutures)
{
future.join();
}
this.dhLevelByLevelWrapper.clear();
this.eventLoop.close();
LOGGER.info("Closed DhWorld of type " + this.environment);
core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java
+20 -5
View File
@@ -29,7 +29,9 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapp
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@@ -127,14 +129,11 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld
public void close()
{
this.networkState.close();
this.dhTickerThread.shutdownNow();
ArrayList<CompletableFuture<Void>> closeFutures = new ArrayList<>();
for (DhClientLevel dhClientLevel : this.levels.values())
{
LOGGER.info("Unloading level [" + dhClientLevel.getLevelWrapper().getDhIdentifier() + "].");
// level wrapper shouldn't be null, but just in case
IClientLevelWrapper clientLevelWrapper = dhClientLevel.getClientLevelWrapper();
if (clientLevelWrapper != null)
@@ -142,7 +141,23 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld
clientLevelWrapper.onUnload();
}
dhClientLevel.close();
// close levels asynchronously to speed up
// shutdown on servers with a lot of levels
CompletableFuture<Void> closeFuture = new CompletableFuture<>();
Thread closeThread = new Thread(() ->
{
dhClientLevel.close();
closeFuture.complete(null);
}, "level shutdown");
closeThread.start();
closeFutures.add(closeFuture);
}
// wait for all the levels to finish closing
for (CompletableFuture<Void> future : closeFutures)
{
future.join();
}
this.levels.clear();
core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java
+1 -1
View File
@@ -64,7 +64,7 @@ public class DhServerWorld extends AbstractDhServerWorld<DhServerLevel>
if (this.dhLevelByLevelWrapper.containsKey(wrapper))
{
LOGGER.info("Unloading level {} ", this.dhLevelByLevelWrapper.get(wrapper));
DhServerLevel level = this.dhLevelByLevelWrapper.get(wrapper);
wrapper.onUnload();
this.dhLevelByLevelWrapper.remove(wrapper).close();
}
core/src/main/resources/assets/distanthorizons/lang/en_us.json
+6
View File
@@ -509,6 +509,10 @@
"Validate Buffer IDs Before Rendering",
"distanthorizons.config.client.advanced.debugging.openGl.validateBufferIdsBeforeRendering.@tooltip":
"Massively reduces FPS. \nShould only be used if mysterious EXCEPTION_ACCESS_VIOLATION crashes are happening in DH's rendering code and you're attempting to troubleshoot it.",
"distanthorizons.config.client.advanced.debugging.openGl.glUploadMode":
"Uploade Mode",
"distanthorizons.config.client.advanced.debugging.openGl.glUploadMode.@tooltip":
"Only for debugging",
@@ -1046,6 +1050,8 @@
"distanthorizons.config.enum.EDhApiGpuUploadMethod.AUTO":
"Auto",
"distanthorizons.config.enum.EDhApiGpuUploadMethod.NONE":
"None",
"distanthorizons.config.enum.EDhApiGpuUploadMethod.BUFFER_STORAGE":
"Buffer storage",
"distanthorizons.config.enum.EDhApiGpuUploadMethod.SUB_DATA":
core/src/main/resources/sqlScripts/0090-sqlite-addAdjacentFullDataColumns.sql
+13
View File
@@ -0,0 +1,13 @@
-- storing adjacent data (IE a single line of data on the +X/-X/+Z/-Z axis)
-- allows for significantly reduced render loading times since we only have to
-- handle part of the adjacent data source vs all of it
alter table FullData add column NorthAdjData BLOB NULL;
--batch--
alter table FullData add column SouthAdjData BLOB NULL;
--batch--
alter table FullData add column EastAdjData BLOB NULL;
--batch--
alter table FullData add column WestAdjData BLOB NULL;
--batch--
core/src/main/resources/sqlScripts/scriptList.txt
+1
View File
@@ -8,3 +8,4 @@
0060-sqlite-createChunkHashTable.sql
0070-sqlite-createBeaconBeamTable.sql
0080-sqlite-addApplyToChildrenColumn.sql
0090-sqlite-addAdjacentFullDataColumns.sql
core/src/test/java/tests/DhFullDataSourceRepoTests.java
+193 -29
View File
@@ -22,12 +22,15 @@ package tests;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.enums.EDhDirection;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil;
import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.ThreadUtil;
import com.seibel.distanthorizons.core.util.objects.DataCorruptedException;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.junit.Assert;
@@ -37,8 +40,11 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
/**
* Can also be used to test if there are memory leaks in SQLite
@@ -78,14 +84,26 @@ public class DhFullDataSourceRepoTests
long pos = DhSectionPos.encode((byte)6, 1, 2);
FullDataPointIdMap dataMapping = new FullDataPointIdMap(pos);
LongArrayList[] fullDataArray = new LongArrayList[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH];
for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++)
Random seededRandom = new Random(3);
for (int arrayIndex = 0; arrayIndex < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; arrayIndex++)
{
fullDataArray[i] = new LongArrayList(1);
fullDataArray[arrayIndex] = new LongArrayList(1);
for (int j = 0; j < 32; j++)
// random column heights so we can differentiate
// columns from each other
int columnCount = Math.abs(seededRandom.nextInt() % 31) + 1;
for (int colIndex = 0; colIndex < columnCount; colIndex++)
{
fullDataArray[i].add(FullDataPointUtil.encode(j, 1, j, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT));
long datapoint = FullDataPointUtil.encode(
colIndex, // id
1, // height
colIndex, // relative min Y
(byte)(colIndex % LodUtil.MAX_MC_LIGHT), // block light
(byte)((colIndex + 2) % LodUtil.MAX_MC_LIGHT) // sky light
);
fullDataArray[arrayIndex].add(datapoint);
}
}
@@ -102,10 +120,23 @@ public class DhFullDataSourceRepoTests
repo.save(originalDto);
// also create format-1 encoded version to ensure backwards compatibility
long posV1 = DhSectionPos.encode((byte) 6, 2, 3);
FullDataSourceV2 dataSourceFormatV1 = FullDataSourceV2.createWithData(posV1, dataMapping, fullDataArray, columnGenStep, columnWorldCompressionMode);
FullDataSourceV2DTO dtoFormatV1 = FullDataSourceV2DTO.CreateFromDataSource(dataSourceFormatV1, EDhApiDataCompressionMode.LZMA2);
FullDataSourceV2DTO.writeDataSourceDataArrayToBlobV1(
dataSourceFormatV1.dataPoints,
dtoFormatV1.compressedDataByteArray,
EDhApiDataCompressionMode.LZMA2);
dtoFormatV1.dataFormatVersion = FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA;
repo.save(dtoFormatV1);
//=========================//
// assert DTO data is the same //
//=========================//
//=======================//
// confirm DTO data is //
// the same after saving //
//=======================//
FullDataSourceV2DTO savedDto = repo.getByKey(pos);
@@ -115,22 +146,124 @@ public class DhFullDataSourceRepoTests
assertArraysAreEqual(originalDto.compressedColumnGenStepByteArray, savedDto.compressedColumnGenStepByteArray);
assertArraysAreEqual(originalDto.compressedWorldCompressionModeByteArray, savedDto.compressedWorldCompressionModeByteArray);
assertArraysAreEqual(originalDto.compressedNorthAdjDataByteArray, savedDto.compressedNorthAdjDataByteArray);
assertArraysAreEqual(originalDto.compressedSouthAdjDataByteArray, savedDto.compressedSouthAdjDataByteArray);
assertArraysAreEqual(originalDto.compressedEastAdjDataByteArray, savedDto.compressedEastAdjDataByteArray);
assertArraysAreEqual(originalDto.compressedWestAdjDataByteArray, savedDto.compressedWestAdjDataByteArray);
//====================================//
// assert dataSource data is the same //
//====================================//
FullDataSourceV2 savedDataSource = savedDto.createUnitTestDataSource();
//========================//
// confirm data source is //
// the same after saving //
//========================//
Assert.assertNotNull("Failed to create DataSource", savedDataSource);
Assert.assertEquals("Pos mismatch", originalDataSource.getPos(), savedDataSource.getPos());
assertArraysAreEqual(originalDataSource.columnGenerationSteps, savedDataSource.columnGenerationSteps);
assertArraysAreEqual(originalDataSource.columnWorldCompressionMode, savedDataSource.columnWorldCompressionMode);
Assert.assertTrue(originalDataSource.dataPoints.length == savedDataSource.dataPoints.length);
for (int i = 0; i < FullDataSourceV2.WIDTH*FullDataSourceV2.WIDTH; i++)
try (FullDataSourceV2 savedDataSource = savedDto.createUnitTestDataSource())
{
assertArraysAreEqual(originalDataSource.dataPoints[i], savedDataSource.dataPoints[i]);
Assert.assertNotNull("Failed to create DataSource", savedDataSource);
Assert.assertEquals("Pos mismatch", originalDataSource.getPos(), savedDataSource.getPos());
assertArraysAreEqual(originalDataSource.columnGenerationSteps, savedDataSource.columnGenerationSteps);
assertArraysAreEqual(originalDataSource.columnWorldCompressionMode, savedDataSource.columnWorldCompressionMode);
Assert.assertEquals(originalDataSource.dataPoints.length, savedDataSource.dataPoints.length);
for (int x = 0; x < FullDataSourceV2.WIDTH; x++)
{
for (int z = 0; z < FullDataSourceV2.WIDTH; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
assertArraysAreEqual("Saved data column at rel pos ["+x+","+z+"] ", originalDataSource.dataPoints[index], savedDataSource.dataPoints[index]);
}
}
}
// check that we have proper backwards compatability to V1
try (FullDataSourceV2 savedDataSource = repo.getByKey(posV1).createUnitTestDataSource())
{
Assert.assertNotNull("Failed to create DataSource", savedDataSource);
assertArraysAreEqual(originalDataSource.columnGenerationSteps, savedDataSource.columnGenerationSteps);
assertArraysAreEqual(originalDataSource.columnWorldCompressionMode,
savedDataSource.columnWorldCompressionMode);
Assert.assertTrue(originalDataSource.dataPoints.length == savedDataSource.dataPoints.length);
for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++)
{
assertArraysAreEqual(originalDataSource.dataPoints[i], savedDataSource.dataPoints[i]);
}
}
//==============//
// adjacent DTO //
//==============//
try (FullDataSourceV2DTO adjDto = repo.getAdjByPosAndDirection(pos, EDhDirection.NORTH))
{
assertArraysAreEqual(adjDto.compressedDataByteArray, savedDto.compressedNorthAdjDataByteArray);
}
try (FullDataSourceV2DTO adjDto = repo.getAdjByPosAndDirection(pos, EDhDirection.SOUTH))
{
assertArraysAreEqual(adjDto.compressedDataByteArray, savedDto.compressedSouthAdjDataByteArray);
}
try (FullDataSourceV2DTO adjDto = repo.getAdjByPosAndDirection(pos, EDhDirection.EAST))
{
assertArraysAreEqual(adjDto.compressedDataByteArray, savedDto.compressedEastAdjDataByteArray);
}
try (FullDataSourceV2DTO adjDto = repo.getAdjByPosAndDirection(pos, EDhDirection.WEST))
{
assertArraysAreEqual(adjDto.compressedDataByteArray, savedDto.compressedWestAdjDataByteArray);
}
//======================//
// adjacent datasources //
//======================//
for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS)
{
try (FullDataSourceV2DTO adjDto = repo.getAdjByPosAndDirection(pos, direction);
FullDataSourceV2 adjSource = adjDto.createUnitTestDataSource(direction))
{
long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction);
int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos);
int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos);
int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos);
int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos);
for (int x = minX; x < maxX; x++)
{
for (int z = minZ; z < maxZ; z++)
{
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList adjDataColumn = adjSource.dataPoints[index];
LongArrayList originalDataColumn = originalDataSource.dataPoints[index];
assertArraysAreEqual(adjDataColumn, originalDataColumn);
}
}
for (int x = 0; x < FullDataSourceV2.WIDTH; x++)
{
for (int z = 0; z < FullDataSourceV2.WIDTH; z++)
{
if (x >= minX && x < maxX
&& z >= minZ && z < maxZ)
{
continue;
}
int index = FullDataSourceV2.relativePosToIndex(x, z);
LongArrayList adjDataColumn = adjSource.dataPoints[index];
Assert.assertEquals(1, adjDataColumn.size());
Assert.assertEquals(FullDataPointUtil.EMPTY_DATA_POINT, adjDataColumn.getLong(0));
}
}
}
}
@@ -148,6 +281,9 @@ public class DhFullDataSourceRepoTests
{
System.out.println("Initial save/get success, starting long update test for GC validation...");
AtomicLong lastLogMsTime = new AtomicLong(0);
LongAdder iterateCount = new LongAdder();
int poolSize = Runtime.getRuntime().availableProcessors();
CompletableFuture<?>[] futures = new CompletableFuture[poolSize];
ThreadPoolExecutor pool = ThreadUtil.makeThreadPool(poolSize, "test pool");
@@ -169,21 +305,47 @@ public class DhFullDataSourceRepoTests
}
// new position so each DTO is different and saved to a different row in the DB
long threadPos = DhSectionPos.encode((byte)0, finalThreadIndex, 0);
long threadPos = DhSectionPos.encode((byte)6, finalThreadIndex, 0);
threadDto.pos = threadPos;
repo.save(threadDto); // runs significantly faster if we don't save
Assert.assertNotNull(threadDto);
// run for a long time
for (int j = 0; j < 1_000_000; j++)
for (int j = 0; j < 100_000_000; j++)
{
repo.save(threadDto); // runs significantly faster if we don't save
try (FullDataSourceV2DTO pooledDto = repo.getByKey(threadPos))
{
Assert.assertNotNull(threadDto);
Assert.assertEquals(pooledDto.pos, threadDto.pos);
assertArraysAreEqual(pooledDto.compressedDataByteArray, threadDto.compressedDataByteArray);
assertArraysAreEqual(pooledDto.compressedColumnGenStepByteArray, threadDto.compressedColumnGenStepByteArray);
assertArraysAreEqual(pooledDto.compressedWorldCompressionModeByteArray, threadDto.compressedWorldCompressionModeByteArray);
Assert.assertFalse(pooledDto.compressedDataByteArray.isEmpty());
Assert.assertFalse(pooledDto.compressedColumnGenStepByteArray.isEmpty());
Assert.assertFalse(pooledDto.compressedWorldCompressionModeByteArray.isEmpty());
try (FullDataSourceV2 dataSource = pooledDto.createUnitTestDataSource();
FullDataSourceV2DTO compressedDto = FullDataSourceV2DTO.CreateFromDataSource(dataSource, EDhApiDataCompressionMode.Z_STD))
{
repo.save(compressedDto);
iterateCount.increment();
long time = System.currentTimeMillis();
if (time - lastLogMsTime.get() > 30_000)
{
lastLogMsTime.set(time);
Runtime runtime = Runtime.getRuntime();
long free = runtime.freeMemory();
long total = runtime.totalMemory();
long max = runtime.maxMemory();
System.out.println("count: "+iterateCount.sum()+"\tfree: "+free+"\ttotal: "+total+"\tmax: "+max);
}
}
catch (Exception ignore)
{
}
}
}
}, pool);
@@ -218,15 +380,17 @@ public class DhFullDataSourceRepoTests
}
private static void assertArraysAreEqual(LongArrayList expectedArray, LongArrayList actualArray)
{ assertArraysAreEqual(null, expectedArray, actualArray); }
private static void assertArraysAreEqual(String message, LongArrayList expectedArray, LongArrayList actualArray)
{
Assert.assertEquals("size mismatch", expectedArray.size(), actualArray.size());
Assert.assertEquals(message + "size mismatch", expectedArray.size(), actualArray.size());
for (int i = 0; i < expectedArray.size(); i++)
{
long expectedNumb = expectedArray.getLong(i);
long actualNumb = actualArray.getLong(i);
Assert.assertEquals("value mismatch at index ["+i+"]", expectedNumb, actualNumb);
Assert.assertEquals(message + "value mismatch at index ["+i+"]", expectedNumb, actualNumb);
}
}
core/src/test/java/tests/FullDataMinMaxPosTest.java
+54
View File
@@ -0,0 +1,54 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package tests;
import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil;
import org.junit.Assert;
import org.junit.Test;
public class FullDataMinMaxPosTest
{
@Test
public void EncodeAdjacentMinMaxPosTest()
{
int maxTest = 3;
for (short minX = 0; minX < maxTest; minX++)
{
for (short maxX = 0; maxX < maxTest; maxX++)
{
for (short minZ = 0; minZ < maxTest; minZ++)
{
for (short maxZ = 0; maxZ < maxTest; maxZ++)
{
long encodedPos = FullDataMinMaxPosUtil.encodeAdjMinMaxPos(minX, maxX, minZ, maxZ);
Assert.assertEquals(minX, FullDataMinMaxPosUtil.getAdjMinX(encodedPos));
Assert.assertEquals(maxX, FullDataMinMaxPosUtil.getAdjMaxX(encodedPos));
Assert.assertEquals(minZ, FullDataMinMaxPosUtil.getAdjMinZ(encodedPos));
Assert.assertEquals(maxZ, FullDataMinMaxPosUtil.getAdjMaxZ(encodedPos));
}
}
}
}
}
}
core/src/test/java/tests/VarintTest.java
+94
View File
@@ -0,0 +1,94 @@
/*
* This file is part of the Distant Horizons mod
* licensed under the GNU LGPL v3 License.
*
* Copyright (C) 2020 James Seibel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package tests;
import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
import com.seibel.distanthorizons.core.sql.dto.util.VarintUtil;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import org.junit.Assert;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
public class VarintTest
{
@Test
public void Test()
{
Assert.assertEquals(0x80, 128);
// zig zag encoding is needed for varint handling, so test it first
for (int i = -256; i < 256; i++)
{
//testZigZagEncoding(i);
}
for (int i = -256; i < 256; i++)
{
//testSingleVarint(i);
}
}
private static void testZigZagEncoding(int value)
{
int encodedValue = VarintUtil.zigzagEncode(value);
int decodedValue = VarintUtil.zigzagDecode(encodedValue);
Assert.assertEquals(value, decodedValue);
}
private static void testSingleVarint(int value)
{
// write to stream
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
try (DhDataOutputStream outputStream = new DhDataOutputStream(byteArrayOutputStream, EDhApiDataCompressionMode.UNCOMPRESSED))
{
int encodedValue = VarintUtil.zigzagEncode(value);
VarintUtil.writeVarint(outputStream, encodedValue); // varint requires zig-zag encoding to function
}
catch (IOException e)
{
e.printStackTrace();
Assert.fail("Fail writing varint ["+value+"], error: ["+e.getMessage()+"]");
}
// read stream
byte[] byteArray = byteArrayOutputStream.toByteArray();
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray);
try (DhDataInputStream inputStream = new DhDataInputStream(byteArrayInputStream, EDhApiDataCompressionMode.UNCOMPRESSED))
{
int encodedValue = VarintUtil.readVarint(inputStream);
int decodedValue = VarintUtil.zigzagDecode(encodedValue);
Assert.assertEquals(value, decodedValue);
}
catch (IOException e)
{
e.printStackTrace();
Assert.fail("Fail reading varint ["+value+"], error: ["+e.getMessage()+"]");
}
}
}