Fix FullDataPointIdMap (de)serialization duplicate Entries
Also increase DATA_FORMAT_VERSION 2 -> 3
This commit is contained in:
James Seibel
+175
-54
@@ -1,18 +1,18 @@
|
||||
package com.seibel.distanthorizons.core.dataObjects.fullData;
|
||||
|
||||
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream;
|
||||
import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream;
|
||||
import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper;
|
||||
import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
|
||||
import com.seibel.distanthorizons.core.util.objects.dataStreams.*;
|
||||
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
/**
|
||||
@@ -25,44 +25,92 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
*/
|
||||
public class FullDataPointIdMap
|
||||
{
|
||||
public static final String SEPARATOR_STRING = "_DH-BSW_";
|
||||
private static final Logger LOGGER = LogManager.getLogger();
|
||||
/**
|
||||
* Should only be enabled when debugging.
|
||||
* Has the system check if any duplicate Entries were read/written
|
||||
* when (de)serializing.
|
||||
*/
|
||||
private static final boolean RUN_SERIALIZATION_DUPLICATE_VALIDATION = false;
|
||||
/** Distant Horizons - Block State Wrapper */
|
||||
private static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_";
|
||||
|
||||
|
||||
// FIXME: Improve performance maybe?
|
||||
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
|
||||
|
||||
final ArrayList<Entry> entries = new ArrayList<>();
|
||||
final HashMap<Entry, Integer> idMap = new HashMap<>();
|
||||
/** used when the data point map is running normally */
|
||||
private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
|
||||
|
||||
private Entry getEntry(int id) {
|
||||
lock.readLock().lock();
|
||||
Entry entry = this.entries.get(id);
|
||||
lock.readLock().unlock();
|
||||
/** should only be used for debugging */
|
||||
private final DhSectionPos pos;
|
||||
|
||||
/** The index should be the same as the Entry's ID */
|
||||
private final ArrayList<Entry> entryList = new ArrayList<>();
|
||||
private final HashMap<Entry, Integer> idMap = new HashMap<>();
|
||||
|
||||
|
||||
|
||||
//=============//
|
||||
// constructor //
|
||||
//=============//
|
||||
|
||||
public FullDataPointIdMap(DhSectionPos pos) { this.pos = pos; }
|
||||
|
||||
|
||||
|
||||
//=========//
|
||||
// methods //
|
||||
//=========//
|
||||
|
||||
private Entry getEntry(int id)
|
||||
{
|
||||
this.readWriteLock.readLock().lock();
|
||||
Entry entry;
|
||||
try
|
||||
{
|
||||
entry = this.entryList.get(id);
|
||||
}
|
||||
catch (IndexOutOfBoundsException e)
|
||||
{
|
||||
LOGGER.error("FullData ID Map out of sync for pos: "+this.pos+". ID: ["+id+"] greater than the number of known ID's: ["+this.entryList.size()+"].");
|
||||
throw e;
|
||||
}
|
||||
|
||||
this.readWriteLock.readLock().unlock();
|
||||
return entry;
|
||||
}
|
||||
|
||||
public IBiomeWrapper getBiomeWrapper(int id) {
|
||||
return getEntry(id).biome;
|
||||
}
|
||||
public IBlockStateWrapper getBlockStateWrapper(int id) {
|
||||
return getEntry(id).blockState;
|
||||
}
|
||||
public IBiomeWrapper getBiomeWrapper(int id) { return this.getEntry(id).biome; }
|
||||
public IBlockStateWrapper getBlockStateWrapper(int id) { return this.getEntry(id).blockState; }
|
||||
|
||||
/**
|
||||
* If an entry with the given values already exists nothing will
|
||||
* be added but the existing item's ID will still be returned.
|
||||
*/
|
||||
public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState)); }
|
||||
private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry)
|
||||
public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState), true); }
|
||||
/** @param useWriteLocks should only be false if this method is already in a write lock to prevent unlocking at the wrong time */
|
||||
private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry, boolean useWriteLocks)
|
||||
{
|
||||
lock.writeLock().lock();
|
||||
int result = this.idMap.computeIfAbsent(biomeBlockStateEntry, (entry) -> {
|
||||
int id = this.entries.size();
|
||||
this.entries.add(entry);
|
||||
return id;
|
||||
});
|
||||
lock.writeLock().unlock();
|
||||
return result;
|
||||
if (useWriteLocks) { this.readWriteLock.writeLock().lock(); }
|
||||
|
||||
|
||||
int id;
|
||||
if (this.idMap.containsKey(biomeBlockStateEntry))
|
||||
{
|
||||
// use the existing ID
|
||||
id = this.idMap.get(biomeBlockStateEntry);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Add the new ID
|
||||
id = this.entryList.size();
|
||||
this.entryList.add(biomeBlockStateEntry);
|
||||
this.idMap.put(biomeBlockStateEntry, id);
|
||||
}
|
||||
|
||||
|
||||
if (useWriteLocks) { this.readWriteLock.writeLock().unlock(); }
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
@@ -72,40 +120,91 @@ public class FullDataPointIdMap
|
||||
*/
|
||||
public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap target)
|
||||
{
|
||||
target.lock.readLock().lock();
|
||||
lock.writeLock().lock();
|
||||
ArrayList<Entry> entriesToMerge = target.entries;
|
||||
LOGGER.trace("merging {"+this.pos+", "+this.entryList.size()+"} and {"+target.pos+", "+target.entryList.size()+"}");
|
||||
|
||||
target.readWriteLock.readLock().lock();
|
||||
this.readWriteLock.writeLock().lock();
|
||||
|
||||
ArrayList<Entry> entriesToMerge = target.entryList;
|
||||
int[] remappedEntryIds = new int[entriesToMerge.size()];
|
||||
for (int i = 0; i < entriesToMerge.size(); i++)
|
||||
{
|
||||
remappedEntryIds[i] = this.addIfNotPresentAndGetId(entriesToMerge.get(i));
|
||||
Entry entity = entriesToMerge.get(i);
|
||||
int id = this.addIfNotPresentAndGetId(entity, false);
|
||||
remappedEntryIds[i] = id;
|
||||
}
|
||||
lock.writeLock().unlock();
|
||||
target.lock.readLock().unlock();
|
||||
|
||||
this.readWriteLock.writeLock().unlock();
|
||||
target.readWriteLock.readLock().unlock();
|
||||
|
||||
LOGGER.trace("finished merging {"+this.pos+", "+this.entryList.size()+"} and {"+target.pos+", "+target.entryList.size()+"}");
|
||||
|
||||
return remappedEntryIds;
|
||||
}
|
||||
|
||||
/** Serializes all contained entries into the given stream, formatted in UTF */
|
||||
public void serialize(DhDataOutputStream outputStream) throws IOException
|
||||
{
|
||||
lock.readLock().lock();
|
||||
outputStream.writeInt(this.entries.size());
|
||||
for (Entry entry : this.entries)
|
||||
this.readWriteLock.readLock().lock();
|
||||
outputStream.writeInt(this.entryList.size());
|
||||
|
||||
// only used when debugging
|
||||
HashMap<String, FullDataPointIdMap.Entry> dataPointEntryBySerialization = new HashMap<>();
|
||||
|
||||
for (Entry entry : this.entryList)
|
||||
{
|
||||
outputStream.writeUTF(entry.serialize());
|
||||
String entryString = entry.serialize();
|
||||
outputStream.writeUTF(entryString);
|
||||
|
||||
if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
|
||||
{
|
||||
if (dataPointEntryBySerialization.containsKey(entryString))
|
||||
{
|
||||
LOGGER.error("Duplicate serialized entry found with serial: " + entryString);
|
||||
}
|
||||
if (dataPointEntryBySerialization.containsValue(entry))
|
||||
{
|
||||
LOGGER.error("Duplicate serialized entry found with value: " + entry.serialize());
|
||||
}
|
||||
dataPointEntryBySerialization.put(entryString, entry);
|
||||
}
|
||||
}
|
||||
lock.readLock().unlock();
|
||||
this.readWriteLock.readLock().unlock();
|
||||
|
||||
LOGGER.trace("serialize "+this.pos+" "+this.entryList.size());
|
||||
}
|
||||
|
||||
/** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */
|
||||
public static FullDataPointIdMap deserialize(DhDataInputStream inputStream) throws IOException, InterruptedException
|
||||
public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos) throws IOException, InterruptedException
|
||||
{
|
||||
int entityCount = inputStream.readInt();
|
||||
FullDataPointIdMap newMap = new FullDataPointIdMap();
|
||||
|
||||
// only used when debugging
|
||||
HashMap<String, FullDataPointIdMap.Entry> dataPointEntryBySerialization = new HashMap<>();
|
||||
|
||||
FullDataPointIdMap newMap = new FullDataPointIdMap(pos);
|
||||
for (int i = 0; i < entityCount; i++)
|
||||
{
|
||||
newMap.entries.add(Entry.deserialize(inputStream.readUTF()));
|
||||
String entryString = inputStream.readUTF();
|
||||
Entry newEntry = Entry.deserialize(entryString);
|
||||
newMap.entryList.add(newEntry);
|
||||
|
||||
if (RUN_SERIALIZATION_DUPLICATE_VALIDATION)
|
||||
{
|
||||
if (dataPointEntryBySerialization.containsKey(entryString))
|
||||
{
|
||||
LOGGER.error("Duplicate deserialized entry found with serial: " + entryString);
|
||||
}
|
||||
if (dataPointEntryBySerialization.containsValue(newEntry))
|
||||
{
|
||||
LOGGER.error("Duplicate deserialized entry found with value: " + newEntry.serialize());
|
||||
}
|
||||
dataPointEntryBySerialization.put(entryString, newEntry);
|
||||
}
|
||||
}
|
||||
|
||||
LOGGER.trace("deserialized "+pos+" "+newMap.entryList.size()+"-"+entityCount);
|
||||
|
||||
return newMap;
|
||||
}
|
||||
|
||||
@@ -131,11 +230,15 @@ public class FullDataPointIdMap
|
||||
|
||||
private static final class Entry
|
||||
{
|
||||
public static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class);
|
||||
private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class);
|
||||
|
||||
public final IBiomeWrapper biome;
|
||||
public final IBlockStateWrapper blockState;
|
||||
|
||||
private Integer hashCode = null;
|
||||
|
||||
|
||||
// constructor //
|
||||
|
||||
public Entry(IBiomeWrapper biome, IBlockStateWrapper blockState)
|
||||
{
|
||||
@@ -144,27 +247,45 @@ public class FullDataPointIdMap
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int hashCode() { return Objects.hash(this.biome, this.blockState); }
|
||||
|
||||
// methods //
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other)
|
||||
public int hashCode()
|
||||
{
|
||||
if (other == this)
|
||||
return true;
|
||||
// cache the hash code to improve speed
|
||||
if (this.hashCode == null)
|
||||
{
|
||||
this.hashCode = this.serialize().hashCode();
|
||||
}
|
||||
|
||||
if (!(other instanceof Entry))
|
||||
return false;
|
||||
|
||||
return ((Entry) other).biome.equals(this.biome) && ((Entry) other).blockState.equals(this.blockState);
|
||||
return this.hashCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object otherObj)
|
||||
{
|
||||
if (otherObj == this)
|
||||
return true;
|
||||
|
||||
if (!(otherObj instanceof Entry))
|
||||
return false;
|
||||
|
||||
Entry other = (Entry) otherObj;
|
||||
return other.biome.serialize().equals(this.biome.serialize())
|
||||
&& other.blockState.serialize().equals(this.blockState.serialize());
|
||||
}
|
||||
|
||||
public String serialize() { return this.biome.serialize() + SEPARATOR_STRING + this.blockState.serialize(); }
|
||||
@Override
|
||||
public String toString() { return this.serialize(); }
|
||||
|
||||
|
||||
|
||||
public String serialize() { return this.biome.serialize() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.serialize(); }
|
||||
|
||||
public static Entry deserialize(String str) throws IOException, InterruptedException
|
||||
{
|
||||
String[] stringArray = str.split(SEPARATOR_STRING);
|
||||
String[] stringArray = str.split(BLOCK_STATE_SEPARATOR_STRING);
|
||||
if (stringArray.length != 2)
|
||||
{
|
||||
throw new IOException("Failed to deserialize BiomeBlockStateEntry");
|
||||
|
||||
+2
-1
@@ -3,6 +3,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.accessor;
|
||||
import com.seibel.distanthorizons.core.pos.DhChunkPos;
|
||||
import com.seibel.distanthorizons.core.pos.DhLodPos;
|
||||
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
|
||||
import com.seibel.distanthorizons.core.pos.DhSectionPos;
|
||||
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
|
||||
import com.seibel.distanthorizons.core.util.LodUtil;
|
||||
|
||||
@@ -22,7 +23,7 @@ public class ChunkSizedFullDataAccessor extends FullDataArrayAccessor
|
||||
|
||||
public ChunkSizedFullDataAccessor(DhChunkPos pos)
|
||||
{
|
||||
super(new FullDataPointIdMap(),
|
||||
super(new FullDataPointIdMap(new DhSectionPos(pos)),
|
||||
new long[LodUtil.CHUNK_WIDTH * LodUtil.CHUNK_WIDTH][0],
|
||||
LodUtil.CHUNK_WIDTH);
|
||||
|
||||
|
||||
+3
-4
@@ -37,7 +37,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
|
||||
/** measured in dataPoints */
|
||||
public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
|
||||
|
||||
public static final byte DATA_FORMAT_VERSION = 2;
|
||||
public static final byte DATA_FORMAT_VERSION = 3;
|
||||
/** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */
|
||||
public static final long TYPE_ID = "CompleteFullDataSource".hashCode();
|
||||
|
||||
@@ -54,7 +54,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
|
||||
public static CompleteFullDataSource createEmpty(DhSectionPos pos) { return new CompleteFullDataSource(pos); }
|
||||
private CompleteFullDataSource(DhSectionPos sectionPos)
|
||||
{
|
||||
super(new FullDataPointIdMap(), new long[WIDTH * WIDTH][0], WIDTH);
|
||||
super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH);
|
||||
this.sectionPos = sectionPos;
|
||||
}
|
||||
|
||||
@@ -230,7 +230,6 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
|
||||
{
|
||||
outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
|
||||
this.mapping.serialize(outputStream);
|
||||
|
||||
}
|
||||
@Override
|
||||
public FullDataPointIdMap readIdMappings(long[][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException
|
||||
@@ -241,7 +240,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu
|
||||
throw new IOException("Invalid data content end guard for ID mapping");
|
||||
}
|
||||
|
||||
return FullDataPointIdMap.deserialize(inputStream);
|
||||
return FullDataPointIdMap.deserialize(inputStream, this.sectionPos);
|
||||
}
|
||||
@Override
|
||||
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
|
||||
|
||||
+9
-10
@@ -51,7 +51,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
|
||||
/** aka max detail level */
|
||||
public static final byte MAX_SECTION_DETAIL = SECTION_SIZE_OFFSET + SPARSE_UNIT_DETAIL;
|
||||
|
||||
public static final byte DATA_FORMAT_VERSION = 2;
|
||||
public static final byte DATA_FORMAT_VERSION = 3;
|
||||
/** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */
|
||||
public static final long TYPE_ID = "HighDetailIncompleteFullDataSource".hashCode();
|
||||
|
||||
@@ -85,7 +85,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
|
||||
|
||||
this.sparseData = new FullDataArrayAccessor[this.sectionCount * this.sectionCount];
|
||||
this.chunkPos = sectionPos.getCorner(SPARSE_UNIT_DETAIL);
|
||||
this.mapping = new FullDataPointIdMap();
|
||||
this.mapping = new FullDataPointIdMap(sectionPos);
|
||||
}
|
||||
|
||||
protected HighDetailIncompleteFullDataSource(DhSectionPos sectionPos, FullDataPointIdMap mapping, FullDataArrayAccessor[] data)
|
||||
@@ -349,13 +349,6 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException
|
||||
{
|
||||
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
|
||||
this.mapping.serialize(dataOutputStream);
|
||||
|
||||
}
|
||||
@Override
|
||||
public FullDataPointIdMap readIdMappings(long[][][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException
|
||||
{
|
||||
@@ -368,7 +361,13 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo
|
||||
}
|
||||
|
||||
// deserialize the ID data
|
||||
return FullDataPointIdMap.deserialize(inputStream);
|
||||
return FullDataPointIdMap.deserialize(inputStream, this.sectionPos);
|
||||
}
|
||||
@Override
|
||||
public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException
|
||||
{
|
||||
dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE);
|
||||
this.mapping.serialize(dataOutputStream);
|
||||
}
|
||||
@Override
|
||||
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
|
||||
|
||||
+3
-3
@@ -42,7 +42,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
|
||||
/** measured in dataPoints */
|
||||
public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET);
|
||||
|
||||
public static final byte DATA_FORMAT_VERSION = 2;
|
||||
public static final byte DATA_FORMAT_VERSION = 3;
|
||||
/** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */
|
||||
public static final long TYPE_ID = "LowDetailIncompleteFullDataSource".hashCode();
|
||||
|
||||
@@ -63,7 +63,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
|
||||
public static LowDetailIncompleteFullDataSource createEmpty(DhSectionPos pos) { return new LowDetailIncompleteFullDataSource(pos); }
|
||||
private LowDetailIncompleteFullDataSource(DhSectionPos sectionPos)
|
||||
{
|
||||
super(new FullDataPointIdMap(), new long[WIDTH * WIDTH][0], WIDTH);
|
||||
super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH);
|
||||
LodUtil.assertTrue(sectionPos.sectionDetailLevel > HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL);
|
||||
|
||||
this.sectionPos = sectionPos;
|
||||
@@ -252,7 +252,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp
|
||||
{
|
||||
throw new IOException("invalid ID mapping end guard");
|
||||
}
|
||||
return FullDataPointIdMap.deserialize(inputStream);
|
||||
return FullDataPointIdMap.deserialize(inputStream, this.sectionPos);
|
||||
}
|
||||
@Override
|
||||
public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }
|
||||
|
||||
Reference in New Issue
Block a user