Merge remote-tracking branch 'upstream-core/main'

# Conflicts:
#	core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java
#	core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java
#	core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java
#	core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java
#	core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java
This commit is contained in:
Steveplays28
2023-08-13 22:59:43 +02:00
8 changed files with 224 additions and 88 deletions
@@ -47,6 +47,9 @@ public class SharedApi
ColumnRenderBufferBuilder.shutdownExecutorService();
WorldGenerationQueue.shutdownWorldGenThreadPool();
ChunkToLodBuilder.shutdownExecutorService();
// recommend that the garbage collector cleans up any objects from the old world
System.gc();
}
}
@@ -1,19 +1,19 @@
package com.seibel.distanthorizons.core.dataObjects.fullData;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import com.seibel.distanthorizons.core.util.objects.dataStreams.*;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
@@ -26,44 +26,92 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
*/
public class FullDataPointIdMap
{
public static final String SEPARATOR_STRING = "_DH-BSW_";
private static final Logger LOGGER = LogManager.getLogger();
/**
* Should only be enabled when debugging.
* Has the system check if any duplicate Entries were read/written
* when (de)serializing.
*/
private static final boolean RUN_SERIALIZATION_DUPLICATE_VALIDATION = false;
/** Distant Horizons - Block State Wrapper */
private static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_";
// FIXME: Improve performance maybe?
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
/** used when the data point map is running normally */
private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
final ArrayList<Entry> entries = new ArrayList<>();
final HashMap<Entry, Integer> idMap = new HashMap<>();
/** should only be used for debugging */
private final DhSectionPos pos;
private Entry getEntry(int id) {
lock.readLock().lock();
Entry entry = this.entries.get(id);
lock.readLock().unlock();
/** The index should be the same as the Entry's ID */
private final ArrayList<Entry> entryList = new ArrayList<>();
private final HashMap<Entry, Integer> idMap = new HashMap<>();
//=============//
// constructor //
//=============//
public FullDataPointIdMap(DhSectionPos pos) { this.pos = pos; }
//=========//
// methods //
//=========//
private Entry getEntry(int id)
{
this.readWriteLock.readLock().lock();
Entry entry;
try
{
entry = this.entryList.get(id);
}
catch (IndexOutOfBoundsException e)
{
LOGGER.error("FullData ID Map out of sync for pos: "+this.pos+". ID: ["+id+"] greater than the number of known ID's: ["+this.entryList.size()+"].");
throw e;
}
this.readWriteLock.readLock().unlock();
return entry;
}
public IBiomeWrapper getBiomeWrapper(int id) {
return getEntry(id).biome;
}
public IBlockStateWrapper getBlockStateWrapper(int id) {
return getEntry(id).blockState;
}
public IBiomeWrapper getBiomeWrapper(int id) { return this.getEntry(id).biome; }
public IBlockStateWrapper getBlockStateWrapper(int id) { return this.getEntry(id).blockState; }
/**
* If an entry with the given values already exists nothing will
* be added but the existing item's ID will still be returned.
*/
public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState)); }
private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry)
public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState), true); }
/** @param useWriteLocks should only be false if this method is already in a write lock to prevent unlocking at the wrong time */
private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry, boolean useWriteLocks)
{
lock.writeLock().lock();
int result = this.idMap.computeIfAbsent(biomeBlockStateEntry, (entry) -> {
int id = this.entries.size();
this.entries.add(entry);
return id;
});
lock.writeLock().unlock();
return result;
if (useWriteLocks) { this.readWriteLock.writeLock().lock(); }
int id;
if (this.idMap.containsKey(biomeBlockStateEntry))
{
// use the existing ID
id = this.idMap.get(biomeBlockStateEntry);
}
else
{
// Add the new ID
id = this.entryList.size();
this.entryList.add(biomeBlockStateEntry);
this.idMap.put(biomeBlockStateEntry, id);
}
if (useWriteLocks) { this.readWriteLock.writeLock().unlock(); }
return id;
}
@@ -73,40 +121,91 @@ public class FullDataPointIdMap
*/
public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap target)
{
target.lock.readLock().lock();
lock.writeLock().lock();
ArrayList<Entry> entriesToMerge = target.entries;
LOGGER.trace("merging {"+this.pos+", "+this.entryList.size()+"} and {"+target.pos+", "+target.entryList.size()+"}");
target.readWriteLock.readLock().lock();
this.readWriteLock.writeLock().lock();
ArrayList<Entry> entriesToMerge = target.entryList;
int[] remappedEntryIds = new int[entriesToMerge.size()];
for (int i = 0; i < entriesToMerge.size(); i++)
{
remappedEntryIds[i] = this.addIfNotPresentAndGetId(entriesToMerge.get(i));
Entry entity = entriesToMerge.get(i);
int id = this.addIfNotPresentAndGetId(entity, false);
remappedEntryIds[i] = id;
}
lock.writeLock().unlock();
target.lock.readLock().unlock();
this.readWriteLock.writeLock().unlock();
target.readWriteLock.readLock().unlock();
LOGGER.trace("finished merging {"+this.pos+", "+this.entryList.size()+"} and {"+target.pos+", "+target.entryList.size()+"}");
return remappedEntryIds;
}
/** Serializes all contained entries into the given stream, formatted in UTF */
public void serialize(DhDataOutputStream outputStream, ILevelWrapper levelWrapper) throws IOException
{
lock.readLock().lock();
outputStream.writeInt(this.entries.size());
for (Entry entry : this.entries)
this.readWriteLock.readLock().lock();
outputStream.writeInt(this.entryList.size());
// only used when debugging
HashMap<String, FullDataPointIdMap.Entry> dataPointEntryBySerialization = new HashMap<>();
for (Entry entry : this.entryList)
{
outputStream.writeUTF(entry.serialize(levelWrapper));
String entryString = entry.serialize(levelWrapper);
outputStream.writeUTF(entryString);
if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
{
if (dataPointEntryBySerialization.containsKey(entryString))
{
LOGGER.error("Duplicate serialized entry found with serial: " + entryString);
}
if (dataPointEntryBySerialization.containsValue(entry))
{
LOGGER.error("Duplicate serialized entry found with value: " + entry.serialize());
}
dataPointEntryBySerialization.put(entryString, entry);
}
}
lock.readLock().unlock();
this.readWriteLock.readLock().unlock();
LOGGER.trace("serialize "+this.pos+" "+this.entryList.size());
}
/** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */
public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException
public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos, ILevelWrapper levelWrapper) throws IOException, InterruptedException
{
int entityCount = inputStream.readInt();
FullDataPointIdMap newMap = new FullDataPointIdMap();
// only used when debugging
HashMap<String, FullDataPointIdMap.Entry> dataPointEntryBySerialization = new HashMap<>();
FullDataPointIdMap newMap = new FullDataPointIdMap(pos);
for (int i = 0; i < entityCount; i++)
{
newMap.entries.add(Entry.deserialize(inputStream.readUTF(), levelWrapper));
String entryString = inputStream.readUTF();
Entry newEntry = Entry.deserialize(entryString);
newMap.entryList.add(newEntry);
if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
{
if (dataPointEntryBySerialization.containsKey(entryString))
{
LOGGER.error("Duplicate deserialized entry found with serial: " + entryString);
}
if (dataPointEntryBySerialization.containsValue(newEntry))
{
LOGGER.error("Duplicate deserialized entry found with value: " + newEntry.serialize());
}
dataPointEntryBySerialization.put(entryString, newEntry);
}
}
LOGGER.trace("deserialized "+pos+" "+newMap.entryList.size()+"-"+entityCount);
return newMap;
}
@@ -132,12 +231,16 @@ public class FullDataPointIdMap
private static final class Entry
{
public static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class);
private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class);
public final IBiomeWrapper biome;
public final IBlockStateWrapper blockState;
private Integer hashCode = null;
// constructor //
public Entry(IBiomeWrapper biome, IBlockStateWrapper blockState)
{
this.biome = biome;
@@ -145,27 +248,44 @@ public class FullDataPointIdMap
}
// methods //
@Override
public int hashCode() { return Objects.hash(this.biome, this.blockState); }
@Override
public boolean equals(Object other)
public int hashCode()
{
if (other == this)
return true;
if (!(other instanceof Entry))
return false;
return ((Entry) other).biome.equals(this.biome) && ((Entry) other).blockState.equals(this.blockState);
// cache the hash code to improve speed
if (this.hashCode == null)
{
this.hashCode = this.serialize().hashCode();
}
return this.hashCode;
}
@Override
public boolean equals(Object otherObj)
{
if (otherObj == this)
return true;
if (!(otherObj instanceof Entry))
return false;
Entry other = (Entry) otherObj;
return other.biome.serialize().equals(this.biome.serialize())
&& other.blockState.serialize().equals(this.blockState.serialize());
}
@Override
public String toString() { return this.serialize(); }
public String serialize(ILevelWrapper levelWrapper) { return this.biome.serialize(levelWrapper) + SEPARATOR_STRING + this.blockState.serialize(levelWrapper); }
public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, InterruptedException
{
String[] stringArray = str.split(SEPARATOR_STRING);
String[] stringArray = str.split(BLOCK_STATE_SEPARATOR_STRING);
if (stringArray.length != 2)
{
throw new IOException("Failed to deserialize BiomeBlockStateEntry");
@@ -3,6 +3,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.accessor;
import com.seibel.distanthorizons.core.pos.DhChunkPos;
import com.seibel.distanthorizons.core.pos.DhLodPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
@@ -22,7 +23,7 @@ public class ChunkSizedFullDataAccessor extends FullDataArrayAccessor
public ChunkSizedFullDataAccessor(DhChunkPos pos)
{
super(new FullDataPointIdMap(),
super(new FullDataPointIdMap(new DhSectionPos(pos)),
new long[LodUtil.CHUNK_WIDTH * LodUtil.CHUNK_WIDTH][0],
LodUtil.CHUNK_WIDTH);
@@ -39,7 +39,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
/** measured in dataPoints */
public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
public static final byte DATA_FORMAT_VERSION = 2;
public static final byte DATA_FORMAT_VERSION = 3;
/** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */
public static final long TYPE_ID = "CompleteFullDataSource".hashCode();
@@ -56,7 +56,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
public static CompleteFullDataSource createEmpty(DhSectionPos pos) { return new CompleteFullDataSource(pos); }
private CompleteFullDataSource(DhSectionPos sectionPos)
{
super(new FullDataPointIdMap(), new long[WIDTH * WIDTH][0], WIDTH);
super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH);
this.sectionPos = sectionPos;
}
@@ -232,7 +232,6 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
{
outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(outputStream, levelWrapper);
}
@Override
public FullDataPointIdMap readIdMappings(long[][] dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException
@@ -244,6 +243,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
}
return FullDataPointIdMap.deserialize(inputStream, levelWrapper);
return FullDataPointIdMap.deserialize(inputStream, this.sectionPos);
}
@Override
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
@@ -382,7 +382,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
@Override
public long getTypeId() { return TYPE_ID; }
@Override
public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; }
@@ -52,7 +52,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
/** aka max detail level */
public static final byte MAX_SECTION_DETAIL = SECTION_SIZE_OFFSET + SPARSE_UNIT_DETAIL;
public static final byte DATA_FORMAT_VERSION = 2;
public static final byte DATA_FORMAT_VERSION = 3;
/** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */
public static final long TYPE_ID = "HighDetailIncompleteFullDataSource".hashCode();
@@ -86,7 +86,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
this.sparseData = new FullDataArrayAccessor[this.sectionCount * this.sectionCount];
this.chunkPos = sectionPos.getCorner(SPARSE_UNIT_DETAIL);
this.mapping = new FullDataPointIdMap();
this.mapping = new FullDataPointIdMap(sectionPos);
}
protected HighDetailIncompleteFullDataSource(DhSectionPos sectionPos, FullDataPointIdMap mapping, FullDataArrayAccessor[] data)
@@ -348,15 +348,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
}
}
}
@Override
public void writeIdMappings(DhDataOutputStream dataOutputStream, ILevelWrapper levelWrapper) throws IOException
{
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(dataOutputStream, levelWrapper);
}
@Override
public FullDataPointIdMap readIdMappings(long[][][] dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException
{
@@ -369,8 +361,16 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
}
// deserialize the ID data
return FullDataPointIdMap.deserialize(inputStream, levelWrapper);
return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper);
}
@Override
public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException
{
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
this.mapping.serialize(dataOutputStream);
}
@Override
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
@@ -404,10 +404,10 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
public DhSectionPos getSectionPos() { return this.sectionPos; }
@Override
public byte getDataDetailLevel() { return (byte) (this.sectionPos.sectionDetailLevel - SECTION_SIZE_OFFSET); }
@Override
public long getTypeId() { return TYPE_ID; }
@Override
public byte getBinaryDataFormatVersion() { return DATA_FORMAT_VERSION; }
@@ -43,7 +43,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
/** measured in dataPoints */
public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
public static final byte DATA_FORMAT_VERSION = 2;
public static final byte DATA_FORMAT_VERSION = 3;
/** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */
public static final long TYPE_ID = "LowDetailIncompleteFullDataSource".hashCode();
@@ -64,7 +64,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
public static LowDetailIncompleteFullDataSource createEmpty(DhSectionPos pos) { return new LowDetailIncompleteFullDataSource(pos); }
private LowDetailIncompleteFullDataSource(DhSectionPos sectionPos)
{
super(new FullDataPointIdMap(), new long[WIDTH * WIDTH][0], WIDTH);
super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH);
LodUtil.assertTrue(sectionPos.sectionDetailLevel > HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL);
this.sectionPos = sectionPos;
@@ -253,7 +253,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
{
throw new IOException("invalid ID mapping end guard");
}
return FullDataPointIdMap.deserialize(inputStream, levelWrapper);
return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper);
}
@Override
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
@@ -9,10 +9,8 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrappe
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* This logic was roughly based on
@@ -43,8 +41,13 @@ public class DhLightingEngine
DhChunkPos centerChunkPos = centerChunk.getChunkPos();
HashMap<DhChunkPos, IChunkWrapper> chunksByChunkPos = new HashMap<>(9);
LinkedList<LightPos> blockLightPosQueue = new LinkedList<>();
LinkedList<LightPos> skyLightPosQueue = new LinkedList<>();
// from some quick testing on James' part,
// these initial capacities should be big enough to fit most lighting jobs
// with a bit of room to spare
ArrayList<LightPos> blockLightPosQueue = new ArrayList<>(40_000); // when tested with a normal 1.20 world James saw a max of: 36_709
ArrayList<LightPos> skyLightPosQueue = new ArrayList<>(3_000); // when tested with a normal 1.20 world James saw a max of: 2355
// generate the list of chunk pos we need,
// currently a 3x3 grid
@@ -144,16 +147,21 @@ public class DhLightingEngine
/** Applies each {@link LightPos} from the queue to the given set of {@link IChunkWrapper}'s. */
private void propagateLightPosList(
LinkedList<LightPos> lightPosQueue, HashMap<DhChunkPos, IChunkWrapper> chunksByChunkPos,
ArrayList<LightPos> lightPosQueue, HashMap<DhChunkPos, IChunkWrapper> chunksByChunkPos,
IGetLightFunc getLightFunc, ISetLightFunc setLightFunc)
{
// update each light position
while (!lightPosQueue.isEmpty())
{
LightPos lightPos = lightPosQueue.poll();
// since we don't care about the order the positions are processed,
// we can grab the last position instead of the first for a slight performance increase (this way the array doesn't need to be shifted over every loop)
int lastIndex = lightPosQueue.size() - 1;
LightPos lightPos = lightPosQueue.remove(lastIndex);
DhBlockPos pos = lightPos.pos;
int lightValue = lightPos.lightValue;
// propagate the lighting in each cardinal direction, IE: -x, +x, -y, +y, -z, +z
for (EDhDirection direction : EDhDirection.CARDINAL_DIRECTIONS)
{
@@ -48,9 +48,13 @@ public class DhSectionPos implements INetworkObject
public int sectionZ;
public static DhSectionPos zero() { return new DhSectionPos((byte) 0, 0, 0); };
//==============//
// constructors //
//==============//
public DhSectionPos(byte sectionDetailLevel, int sectionX, int sectionZ)
{
this.sectionDetailLevel = sectionDetailLevel;