diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index 74d1e1645..660f597a8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -1,18 +1,18 @@ package com.seibel.distanthorizons.core.dataObjects.fullData; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; -import com.seibel.distanthorizons.core.util.objects.dataStreams.*; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.*; import java.util.ArrayList; import java.util.HashMap; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; /** @@ -25,44 +25,92 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; */ public class FullDataPointIdMap { - public static final String SEPARATOR_STRING = "_DH-BSW_"; + private static final Logger LOGGER = LogManager.getLogger(); + /** + * Should only be enabled when debugging. + * Has the system check if any duplicate Entries were read/written + * when (de)serializing. + */ + private static final boolean RUN_SERIALIZATION_DUPLICATE_VALIDATION = false; + /** Distant Horizons - Block State Wrapper */ + private static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_"; // FIXME: Improve performance maybe? - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - - final ArrayList entries = new ArrayList<>(); - final HashMap idMap = new HashMap<>(); + /** used when the data point map is running normally */ + private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); - private Entry getEntry(int id) { - lock.readLock().lock(); - Entry entry = this.entries.get(id); - lock.readLock().unlock(); + /** should only be used for debugging */ + private final DhSectionPos pos; + + /** The index should be the same as the Entry's ID */ + private final ArrayList entryList = new ArrayList<>(); + private final HashMap idMap = new HashMap<>(); + + + + //=============// + // constructor // + //=============// + + public FullDataPointIdMap(DhSectionPos pos) { this.pos = pos; } + + + + //=========// + // methods // + //=========// + + private Entry getEntry(int id) + { + this.readWriteLock.readLock().lock(); + Entry entry; + try + { + entry = this.entryList.get(id); + } + catch (IndexOutOfBoundsException e) + { + LOGGER.error("FullData ID Map out of sync for pos: "+this.pos+". ID: ["+id+"] greater than the number of known ID's: ["+this.entryList.size()+"]."); + throw e; + } + + this.readWriteLock.readLock().unlock(); return entry; } - public IBiomeWrapper getBiomeWrapper(int id) { - return getEntry(id).biome; - } - public IBlockStateWrapper getBlockStateWrapper(int id) { - return getEntry(id).blockState; - } + public IBiomeWrapper getBiomeWrapper(int id) { return this.getEntry(id).biome; } + public IBlockStateWrapper getBlockStateWrapper(int id) { return this.getEntry(id).blockState; } /** * If an entry with the given values already exists nothing will * be added but the existing item's ID will still be returned. */ - public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState)); } - private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry) + public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState), true); } + /** @param useWriteLocks should only be false if this method is already in a write lock to prevent unlocking at the wrong time */ + private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry, boolean useWriteLocks) { - lock.writeLock().lock(); - int result = this.idMap.computeIfAbsent(biomeBlockStateEntry, (entry) -> { - int id = this.entries.size(); - this.entries.add(entry); - return id; - }); - lock.writeLock().unlock(); - return result; + if (useWriteLocks) { this.readWriteLock.writeLock().lock(); } + + + int id; + if (this.idMap.containsKey(biomeBlockStateEntry)) + { + // use the existing ID + id = this.idMap.get(biomeBlockStateEntry); + } + else + { + // Add the new ID + id = this.entryList.size(); + this.entryList.add(biomeBlockStateEntry); + this.idMap.put(biomeBlockStateEntry, id); + } + + + if (useWriteLocks) { this.readWriteLock.writeLock().unlock(); } + + return id; } @@ -72,40 +120,91 @@ public class FullDataPointIdMap */ public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap target) { - target.lock.readLock().lock(); - lock.writeLock().lock(); - ArrayList entriesToMerge = target.entries; + LOGGER.trace("merging {"+this.pos+", "+this.entryList.size()+"} and {"+target.pos+", "+target.entryList.size()+"}"); + + target.readWriteLock.readLock().lock(); + this.readWriteLock.writeLock().lock(); + + ArrayList entriesToMerge = target.entryList; int[] remappedEntryIds = new int[entriesToMerge.size()]; for (int i = 0; i < entriesToMerge.size(); i++) { - remappedEntryIds[i] = this.addIfNotPresentAndGetId(entriesToMerge.get(i)); + Entry entity = entriesToMerge.get(i); + int id = this.addIfNotPresentAndGetId(entity, false); + remappedEntryIds[i] = id; } - lock.writeLock().unlock(); - target.lock.readLock().unlock(); + + this.readWriteLock.writeLock().unlock(); + target.readWriteLock.readLock().unlock(); + + LOGGER.trace("finished merging {"+this.pos+", "+this.entryList.size()+"} and {"+target.pos+", "+target.entryList.size()+"}"); + return remappedEntryIds; } /** Serializes all contained entries into the given stream, formatted in UTF */ public void serialize(DhDataOutputStream outputStream) throws IOException { - lock.readLock().lock(); - outputStream.writeInt(this.entries.size()); - for (Entry entry : this.entries) + this.readWriteLock.readLock().lock(); + outputStream.writeInt(this.entryList.size()); + + // only used when debugging + HashMap dataPointEntryBySerialization = new HashMap<>(); + + for (Entry entry : this.entryList) { - outputStream.writeUTF(entry.serialize()); + String entryString = entry.serialize(); + outputStream.writeUTF(entryString); + + if (RUN_SERIALIZATION_DUPLICATE_VALIDATION) + { + if (dataPointEntryBySerialization.containsKey(entryString)) + { + LOGGER.error("Duplicate serialized entry found with serial: " + entryString); + } + if (dataPointEntryBySerialization.containsValue(entry)) + { + LOGGER.error("Duplicate serialized entry found with value: " + entry.serialize()); + } + dataPointEntryBySerialization.put(entryString, entry); + } } - lock.readLock().unlock(); + this.readWriteLock.readLock().unlock(); + + LOGGER.trace("serialize "+this.pos+" "+this.entryList.size()); } /** Creates a new IdBiomeBlockStateMap from the given UTF formatted stream */ - public static FullDataPointIdMap deserialize(DhDataInputStream inputStream) throws IOException, InterruptedException + public static FullDataPointIdMap deserialize(DhDataInputStream inputStream, DhSectionPos pos) throws IOException, InterruptedException { int entityCount = inputStream.readInt(); - FullDataPointIdMap newMap = new FullDataPointIdMap(); + + // only used when debugging + HashMap dataPointEntryBySerialization = new HashMap<>(); + + FullDataPointIdMap newMap = new FullDataPointIdMap(pos); for (int i = 0; i < entityCount; i++) { - newMap.entries.add(Entry.deserialize(inputStream.readUTF())); + String entryString = inputStream.readUTF(); + Entry newEntry = Entry.deserialize(entryString); + newMap.entryList.add(newEntry); + + if (RUN_SERIALIZATION_DUPLICATE_VALIDATION) + { + if (dataPointEntryBySerialization.containsKey(entryString)) + { + LOGGER.error("Duplicate deserialized entry found with serial: " + entryString); + } + if (dataPointEntryBySerialization.containsValue(newEntry)) + { + LOGGER.error("Duplicate deserialized entry found with value: " + newEntry.serialize()); + } + dataPointEntryBySerialization.put(entryString, newEntry); + } } + + LOGGER.trace("deserialized "+pos+" "+newMap.entryList.size()+"-"+entityCount); + return newMap; } @@ -131,11 +230,15 @@ public class FullDataPointIdMap private static final class Entry { - public static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class); + private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class); public final IBiomeWrapper biome; public final IBlockStateWrapper blockState; + private Integer hashCode = null; + + + // constructor // public Entry(IBiomeWrapper biome, IBlockStateWrapper blockState) { @@ -144,27 +247,45 @@ public class FullDataPointIdMap } - @Override - public int hashCode() { return Objects.hash(this.biome, this.blockState); } + + // methods // @Override - public boolean equals(Object other) + public int hashCode() { - if (other == this) - return true; + // cache the hash code to improve speed + if (this.hashCode == null) + { + this.hashCode = this.serialize().hashCode(); + } - if (!(other instanceof Entry)) - return false; - - return ((Entry) other).biome.equals(this.biome) && ((Entry) other).blockState.equals(this.blockState); + return this.hashCode; } + @Override + public boolean equals(Object otherObj) + { + if (otherObj == this) + return true; + + if (!(otherObj instanceof Entry)) + return false; + + Entry other = (Entry) otherObj; + return other.biome.serialize().equals(this.biome.serialize()) + && other.blockState.serialize().equals(this.blockState.serialize()); + } - public String serialize() { return this.biome.serialize() + SEPARATOR_STRING + this.blockState.serialize(); } + @Override + public String toString() { return this.serialize(); } + + + + public String serialize() { return this.biome.serialize() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.serialize(); } public static Entry deserialize(String str) throws IOException, InterruptedException { - String[] stringArray = str.split(SEPARATOR_STRING); + String[] stringArray = str.split(BLOCK_STATE_SEPARATOR_STRING); if (stringArray.length != 2) { throw new IOException("Failed to deserialize BiomeBlockStateEntry"); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java index 19f1ddd47..156ab3c79 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java @@ -3,6 +3,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; @@ -22,7 +23,7 @@ public class ChunkSizedFullDataAccessor extends FullDataArrayAccessor public ChunkSizedFullDataAccessor(DhChunkPos pos) { - super(new FullDataPointIdMap(), + super(new FullDataPointIdMap(new DhSectionPos(pos)), new long[LodUtil.CHUNK_WIDTH * LodUtil.CHUNK_WIDTH][0], LodUtil.CHUNK_WIDTH); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 7e3cda75e..77d9e8801 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -37,7 +37,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu /** measured in dataPoints */ public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); - public static final byte DATA_FORMAT_VERSION = 2; + public static final byte DATA_FORMAT_VERSION = 3; /** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */ public static final long TYPE_ID = "CompleteFullDataSource".hashCode(); @@ -54,7 +54,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu public static CompleteFullDataSource createEmpty(DhSectionPos pos) { return new CompleteFullDataSource(pos); } private CompleteFullDataSource(DhSectionPos sectionPos) { - super(new FullDataPointIdMap(), new long[WIDTH * WIDTH][0], WIDTH); + super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH); this.sectionPos = sectionPos; } @@ -230,7 +230,6 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu { outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); this.mapping.serialize(outputStream); - } @Override public FullDataPointIdMap readIdMappings(long[][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException @@ -241,7 +240,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu throw new IOException("Invalid data content end guard for ID mapping"); } - return FullDataPointIdMap.deserialize(inputStream); + return FullDataPointIdMap.deserialize(inputStream, this.sectionPos); } @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java index 5b12f6ed2..0ef41aa3c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java @@ -51,7 +51,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo /** aka max detail level */ public static final byte MAX_SECTION_DETAIL = SECTION_SIZE_OFFSET + SPARSE_UNIT_DETAIL; - public static final byte DATA_FORMAT_VERSION = 2; + public static final byte DATA_FORMAT_VERSION = 3; /** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */ public static final long TYPE_ID = "HighDetailIncompleteFullDataSource".hashCode(); @@ -85,7 +85,7 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo this.sparseData = new FullDataArrayAccessor[this.sectionCount * this.sectionCount]; this.chunkPos = sectionPos.getCorner(SPARSE_UNIT_DETAIL); - this.mapping = new FullDataPointIdMap(); + this.mapping = new FullDataPointIdMap(sectionPos); } protected HighDetailIncompleteFullDataSource(DhSectionPos sectionPos, FullDataPointIdMap mapping, FullDataArrayAccessor[] data) @@ -349,13 +349,6 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo } - @Override - public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException - { - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(dataOutputStream); - - } @Override public FullDataPointIdMap readIdMappings(long[][][] dataPoints, DhDataInputStream inputStream) throws IOException, InterruptedException { @@ -368,7 +361,13 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo } // deserialize the ID data - return FullDataPointIdMap.deserialize(inputStream); + return FullDataPointIdMap.deserialize(inputStream, this.sectionPos); + } + @Override + public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException + { + dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + this.mapping.serialize(dataOutputStream); } @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java index e8ab5cc75..5dfd2f207 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java @@ -42,7 +42,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp /** measured in dataPoints */ public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); - public static final byte DATA_FORMAT_VERSION = 2; + public static final byte DATA_FORMAT_VERSION = 3; /** written to the binary file to mark what {@link IFullDataSource} the binary file corresponds to */ public static final long TYPE_ID = "LowDetailIncompleteFullDataSource".hashCode(); @@ -63,7 +63,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp public static LowDetailIncompleteFullDataSource createEmpty(DhSectionPos pos) { return new LowDetailIncompleteFullDataSource(pos); } private LowDetailIncompleteFullDataSource(DhSectionPos sectionPos) { - super(new FullDataPointIdMap(), new long[WIDTH * WIDTH][0], WIDTH); + super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH); LodUtil.assertTrue(sectionPos.sectionDetailLevel > HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL); this.sectionPos = sectionPos; @@ -252,7 +252,7 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp { throw new IOException("invalid ID mapping end guard"); } - return FullDataPointIdMap.deserialize(inputStream); + return FullDataPointIdMap.deserialize(inputStream, this.sectionPos); } @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); }