From 3681d50eb236cc6b056fcdba372fc3d32a166ec5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 1 Nov 2025 08:42:25 -0400 Subject: [PATCH 01/24] minor comment cleanup --- .../api/enums/config/EDhApiDataCompressionMode.java | 3 ++- .../core/dataObjects/fullData/sources/FullDataSourceV2.java | 4 +--- .../seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java | 5 +---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java index 9953ae594..e121b0eda 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java @@ -22,7 +22,8 @@ package com.seibel.distanthorizons.api.enums.config; /** * UNCOMPRESSED
* LZ4
- * XZ

+ * Z_STD
+ * LZMA2

* * Note: speed and compression ratios are examples * and should only be used for estimated comparisons. diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 0fdee4d55..7838d941e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -116,9 +116,7 @@ public class FullDataSourceV2 /** * stored x/z, y
- * The y data should be sorted from top to bottom
- * TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately - * there's something in the render data logic that expects this order so we can't change it right now + * The y data should be sorted from top to bottom */ public final LongArrayList[] dataPoints; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index 9e68e8c02..76bcfb536 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -716,10 +716,7 @@ public abstract class AbstractDhRepo> implemen - /** - * should NOT start with WHERE - * Example: TODO - */ + /** should not start with WHERE */ protected abstract String CreateParameterizedWhereString(); protected void setPreparedStatementWhereClause(PreparedStatement statement, TKey key) throws SQLException { this.setPreparedStatementWhereClause(statement, 1, key); } From e355366ffc3a84b89355d8b0c8bb5bed82c0171a Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 1 Nov 2025 09:06:53 -0400 Subject: [PATCH 02/24] Clean up EDhDirection --- .../render/bufferBuilding/BufferQuad.java | 12 +- .../ColumnRenderBufferBuilder.java | 8 +- .../render/bufferBuilding/LodQuadBuilder.java | 4 +- .../core/enums/EDhDirection.java | 593 +++--------------- .../core/generation/DhLightingEngine.java | 2 +- .../core/pos/DhSectionPos.java | 4 +- .../core/pos/blockPos/DhBlockPos.java | 4 +- .../core/pos/blockPos/DhBlockPosMutable.java | 2 +- .../core/render/LodQuadTree.java | 4 +- .../core/render/LodRenderSection.java | 4 +- 10 files changed, 123 insertions(+), 514 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java index bc25248fd..9bd290581 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java @@ -35,7 +35,7 @@ public final class BufferQuad public static final int NORMAL_MAX_QUAD_WIDTH = 2048; /** * The maximum number of blocks wide a quad can be - * when {@link Config.Client.Advanced.Graphics.AdvancedGraphics#earthCurveRatio earthCurveRatio} + * when {@link Config.Client.Advanced.Graphics.Experimental#earthCurveRatio earthCurveRatio} * is enabled. */ public static final int MAX_QUAD_WIDTH_FOR_EARTH_CURVATURE = LodUtil.CHUNK_WIDTH; @@ -99,7 +99,7 @@ public final class BufferQuad if (compareDirection == BufferMergeDirectionEnum.EastWest) { - switch (this.direction.getAxis()) + switch (this.direction.axis) { case X: return threeDimensionalCompare(this.x, this.y, this.z, quad.x, quad.y, quad.z); @@ -109,12 +109,12 @@ public final class BufferQuad return threeDimensionalCompare(this.z, this.y, this.x, quad.z, quad.y, quad.x); default: - throw new IllegalArgumentException("Invalid Axis enum: " + this.direction.getAxis()); + throw new IllegalArgumentException("Invalid Axis enum: [" + this.direction.axis + "]."); } } else { - switch (this.direction.getAxis()) + switch (this.direction.axis) { case X: return threeDimensionalCompare(this.x, this.z, this.y, quad.x, quad.z, quad.y); @@ -124,7 +124,7 @@ public final class BufferQuad return threeDimensionalCompare(this.z, this.x, this.y, quad.z, quad.x, quad.y); default: - throw new IllegalArgumentException("Invalid Axis enum: " + this.direction.getAxis()); + throw new IllegalArgumentException("Invalid Axis enum: [" + this.direction.axis + "]."); } } } @@ -169,7 +169,7 @@ public final class BufferQuad short thisParallelCompareStartPos; // edge parallel to the merge direction short otherPerpendicularCompareStartPos; short otherParallelCompareStartPos; - switch (this.direction.getAxis()) + switch (this.direction.axis) { default: // shouldn't normally happen, just here to make the compiler happy case X: diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index fe7f256ea..a75b1566d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -150,13 +150,13 @@ public class ColumnRenderBufferBuilder // get adjacent render data columns // //==================================// - ColumnArrayView[] adjColumnViews = new ColumnArrayView[EDhDirection.ADJ_DIRECTIONS.length]; - for (EDhDirection lodDirection : EDhDirection.ADJ_DIRECTIONS) + ColumnArrayView[] adjColumnViews = new ColumnArrayView[EDhDirection.CARDINAL_COMPASS.length]; + for (EDhDirection lodDirection : EDhDirection.CARDINAL_COMPASS) { try { - int xAdj = relX + lodDirection.getNormal().x; - int zAdj = relZ + lodDirection.getNormal().z; + int xAdj = relX + lodDirection.normal.x; + int zAdj = relZ + lodDirection.normal.z; boolean isCrossRenderSourceBoundary = (xAdj < 0 || xAdj >= ColumnRenderSource.SECTION_SIZE) || (zAdj < 0 || zAdj >= ColumnRenderSource.SECTION_SIZE); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index b55392db6..463e171b8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -304,7 +304,7 @@ public class LodQuadBuilder short widthEastWest = quad.widthEastWest; short widthNorthSouth = quad.widthNorthSouthOrUpDown; byte normalIndex = (byte) quad.direction.ordinal(); - EDhDirection.Axis axis = quad.direction.getAxis(); + EDhDirection.Axis axis = quad.direction.axis; for (int i = 0; i < quadBase.length; i++) { short dx, dy, dz; @@ -352,7 +352,7 @@ public class LodQuadBuilder if (this.grassSideRenderingMode != EDhApiGrassSideRendering.AS_GRASS) { // only change the vertex color if it's on the side or bottom - if (quad.direction.getAxis().isHorizontal() || quad.direction == EDhDirection.DOWN) + if (quad.direction.axis.isHorizontal() || quad.direction == EDhDirection.DOWN) { if (this.grassSideRenderingMode == EDhApiGrassSideRendering.AS_DIRT // if we want the color to fade, only apply the dirt color to the bottom vertices diff --git a/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java b/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java index 0b21d4d77..337a1b544 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java @@ -19,496 +19,162 @@ package com.seibel.distanthorizons.core.enums; -import java.util.Arrays; -import java.util.Locale; -import java.util.Map; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import com.seibel.distanthorizons.core.util.math.Vec3i; /** - * An (almost) exact copy of Minecraft's - * Direction enum.

- * * Up
* Down
* North
* South
* East
* West
- * - * @author James Seibel - * @version 2021-11-13 */ public enum EDhDirection { /** negative Y */ - DOWN(0, 1, -1, "down", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Y, new Vec3i(0, -1, 0)), + DOWN("down", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Y, new Vec3i(0, -1, 0)), /** positive Y */ - UP(1, 0, -1, "up", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Y, new Vec3i(0, 1, 0)), + UP("up", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Y, new Vec3i(0, 1, 0)), /** negative Z */ - NORTH(2, 3, 2, "north", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, -1)), + NORTH("north", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, -1)), /** positive Z */ - SOUTH(3, 2, 0, "south", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, 1)), + SOUTH("south", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, 1)), /** negative X */ - WEST(4, 5, 1, "west", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.X, new Vec3i(-1, 0, 0)), + WEST("west", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.X, new Vec3i(-1, 0, 0)), /** positive X */ - EAST(5, 4, 3, "east", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.X, new Vec3i(1, 0, 0)); + EAST("east", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.X, new Vec3i(1, 0, 0)); - /** - * Up, Down, West, East, North, South
- * Similar to {@link EDhDirection#OPPOSITE_DIRECTIONS}, just with a different order - */ - public static final EDhDirection[] CARDINAL_DIRECTIONS = new EDhDirection[]{ + + /** Up, Down, West, East, North, South */ + public static final EDhDirection[] ALL = new EDhDirection[] { EDhDirection.UP, EDhDirection.DOWN, EDhDirection.WEST, EDhDirection.EAST, EDhDirection.NORTH, - EDhDirection.SOUTH}; + EDhDirection.SOUTH + }; - /** - * Up, Down, South, North, East, West
- * Similar to {@link EDhDirection#CARDINAL_DIRECTIONS}, just with a different order - */ - public static final EDhDirection[] OPPOSITE_DIRECTIONS = new EDhDirection[]{ - EDhDirection.UP, - EDhDirection.DOWN, - EDhDirection.SOUTH, - EDhDirection.NORTH, - EDhDirection.EAST, - EDhDirection.WEST}; - - /** North, South, East, West */ // TODO rename to state this is just X/Z or flat directions - public static final EDhDirection[] ADJ_DIRECTIONS = new EDhDirection[]{ + /** North, South, East, West */ + public static final EDhDirection[] CARDINAL_COMPASS = new EDhDirection[] { EDhDirection.EAST, EDhDirection.WEST, EDhDirection.SOUTH, - EDhDirection.NORTH}; - -// private final int data3d; -// private final int oppositeIndex; -// private final int data2d; + EDhDirection.NORTH + }; - private final String name; - private final EDhDirection.Axis axis; - private final EDhDirection.AxisDirection axisDirection; - private final Vec3i normal; - private static final EDhDirection[] VALUES = values(); - private static final Map BY_NAME = Arrays.stream(VALUES).collect(Collectors.toMap(EDhDirection::getName, (p_199787_0_) -> + + public final String name; + public final EDhDirection.Axis axis; + public final EDhDirection.AxisDirection axisDirection; + public final Vec3i normal; + + + + //=============// + // constructor // + //=============// + + EDhDirection(String name, EDhDirection.AxisDirection axisDirection, EDhDirection.Axis axis, Vec3i normal) { - return p_199787_0_; - })); - -// private static final LodDirection[] BY_3D_DATA = Arrays.stream(VALUES).sorted(Comparator.comparingInt((p_199790_0_) -> -// { -// return p_199790_0_.data3d; -// })).toArray((p_199788_0_) -> -// { -// return new LodDirection[p_199788_0_]; -// }); -// -// private static final LodDirection[] BY_2D_DATA = Arrays.stream(VALUES).filter((p_199786_0_) -> -// { -// return p_199786_0_.getAxis().isHorizontal(); -// }).sorted(Comparator.comparingInt((p_199789_0_) -> -// { -// return p_199789_0_.data2d; -// })).toArray((p_199791_0_) -> -// { -// return new LodDirection[p_199791_0_]; -// }); - -// private static final Long2ObjectMap BY_NORMAL = Arrays.stream(VALUES).collect(Collectors.toMap((p_218385_0_) -> -// { -// return (new BlockPos(p_218385_0_.getNormal())).asLong(); -// }, (p_218384_0_) -> -// { -// return p_218384_0_; -// }, (p_218386_0_, p_218386_1_) -> -// { -// throw new IllegalArgumentException("Duplicate keys"); -// }, Long2ObjectOpenHashMap::new)); - - - - EDhDirection(int p_i46016_3_, int p_i46016_4_, int p_i46016_5_, String p_i46016_6_, EDhDirection.AxisDirection p_i46016_7_, EDhDirection.Axis p_i46016_8_, Vec3i p_i46016_9_) - { -// this.data3d = p_i46016_3_; -// this.data2d = p_i46016_5_; -// this.oppositeIndex = p_i46016_4_; - this.name = p_i46016_6_; - this.axis = p_i46016_8_; - this.axisDirection = p_i46016_7_; - this.normal = p_i46016_9_; + this.name = name; + this.axis = axis; + this.axisDirection = axisDirection; + this.normal = normal; } - - - - -// public static LodDirection[] orderedByNearest(Entity p_196054_0_) -// { -// float f = p_196054_0_.getViewXRot(1.0F) * ((float) Math.PI / 180F); -// float f1 = -p_196054_0_.getViewYRot(1.0F) * ((float) Math.PI / 180F); -// float f2 = MathHelper.sin(f); -// float f3 = MathHelper.cos(f); -// float f4 = MathHelper.sin(f1); -// float f5 = MathHelper.cos(f1); -// boolean flag = f4 > 0.0F; -// boolean flag1 = f2 < 0.0F; -// boolean flag2 = f5 > 0.0F; -// float f6 = flag ? f4 : -f4; -// float f7 = flag1 ? -f2 : f2; -// float f8 = flag2 ? f5 : -f5; -// float f9 = f6 * f3; -// float f10 = f8 * f3; -// LodDirection lodDirection = flag ? EAST : WEST; -// LodDirection direction1 = flag1 ? UP : DOWN; -// LodDirection direction2 = flag2 ? SOUTH : NORTH; -// if (f6 > f8) -// { -// if (f7 > f9) -// { -// return makeDirectionArray(direction1, lodDirection, direction2); -// } -// else -// { -// return f10 > f7 ? makeDirectionArray(lodDirection, direction2, direction1) : makeDirectionArray(lodDirection, direction1, direction2); -// } -// } -// else if (f7 > f10) -// { -// return makeDirectionArray(direction1, direction2, lodDirection); -// } -// else -// { -// return f9 > f7 ? makeDirectionArray(direction2, lodDirection, direction1) : makeDirectionArray(direction2, direction1, lodDirection); -// } -// } - -// private static LodDirection[] makeDirectionArray(LodDirection p_196053_0_, LodDirection p_196053_1_, LodDirection p_196053_2_) -// { -// return new LodDirection[] { p_196053_0_, p_196053_1_, p_196053_2_, p_196053_2_.getOpposite(), p_196053_1_.getOpposite(), p_196053_0_.getOpposite() }; -// } - -// public static LodDirection rotate(Mat4f p_229385_0_, LodDirection p_229385_1_) -// { -// Vec3i Vec3i = p_229385_1_.getNormal(); -// Vector4f vector4f = new Vector4f(Vec3i.getX(), Vec3i.getY(), Vec3i.getZ(), 0.0F); -// vector4f.transform(p_229385_0_); -// return getNearest(vector4f.x(), vector4f.y(), vector4f.z()); -// } - -// public Quaternion getRotation() -// { -// Quaternion quaternion = Vector3f.XP.rotationDegrees(90.0F); -// switch (this) -// { -// case DOWN: -// return Vector3f.XP.rotationDegrees(180.0F); -// case UP: -// return Quaternion.ONE.copy(); -// case NORTH: -// quaternion.mul(Vector3f.ZP.rotationDegrees(180.0F)); -// return quaternion; -// case SOUTH: -// return quaternion; -// case WEST: -// quaternion.mul(Vector3f.ZP.rotationDegrees(90.0F)); -// return quaternion; -// case EAST: -// default: -// quaternion.mul(Vector3f.ZP.rotationDegrees(-90.0F)); -// return quaternion; -// } -// } - -// public int get3DDataValue() -// { -// return this.data3d; -// } -// -// public int get2DDataValue() -// { -// return this.data2d; -// } - public EDhDirection.AxisDirection getAxisDirection() - { - return this.axisDirection; - } - -// public LodDirection getOpposite() -// { -// return from3DDataValue(this.oppositeIndex); -// } - public EDhDirection getClockWise() - { - switch (this) + + //=========// + // methods // + //=========// + + public EDhDirection opposite() + { + switch(this) { + case UP: + return EDhDirection.DOWN; + case DOWN: + return EDhDirection.UP; + case NORTH: - return EAST; + return EDhDirection.SOUTH; case SOUTH: - return WEST; - case WEST: - return NORTH; + return EDhDirection.NORTH; + case EAST: - return SOUTH; - default: - throw new IllegalStateException("Unable to get Y-rotated facing of " + this); - } - } - - public EDhDirection getCounterClockWise() - { - switch (this) - { - case NORTH: - return WEST; - case SOUTH: - return EAST; + return EDhDirection.WEST; case WEST: - return SOUTH; - case EAST: - return NORTH; + return EDhDirection.EAST; + default: - throw new IllegalStateException("Unable to get CCW facing of " + this); + throw new IllegalArgumentException(); } } - public String getName() - { - return this.name; - } - public EDhDirection.Axis getAxis() - { - return this.axis; - } + @Override + public String toString() { return this.name; } - public static EDhDirection byName(String name) - { - return name == null ? null : BY_NAME.get(name.toLowerCase(Locale.ROOT)); - } - -// public static LodDirection from3DDataValue(int p_82600_0_) -// { -// return BY_3D_DATA[MathHelper.abs(p_82600_0_ % BY_3D_DATA.length)]; -// } -// -// public static LodDirection from2DDataValue(int p_176731_0_) -// { -// return BY_2D_DATA[MathHelper.abs(p_176731_0_ % BY_2D_DATA.length)]; -// } - -// @Nullable -// public static LodDirection fromNormal(int p_218383_0_, int p_218383_1_, int p_218383_2_) -// { -// return BY_NORMAL.get(BlockPos.asLong(p_218383_0_, p_218383_1_, p_218383_2_)); -// } - -// public static LodDirection fromYRot(double p_176733_0_) -// { -// return from2DDataValue(MathHelper.floor(p_176733_0_ / 90.0D + 0.5D) & 3); -// } - public static EDhDirection fromAxisAndDirection(EDhDirection.Axis p_211699_0_, EDhDirection.AxisDirection p_211699_1_) - { - switch (p_211699_0_) - { - case X: - return p_211699_1_ == EDhDirection.AxisDirection.POSITIVE ? EAST : WEST; - case Y: - return p_211699_1_ == EDhDirection.AxisDirection.POSITIVE ? UP : DOWN; - case Z: - default: - return p_211699_1_ == EDhDirection.AxisDirection.POSITIVE ? SOUTH : NORTH; - } - } - -// public float toYRot() -// { -// return (this.data2d & 3) * 90; -// } - -// public static LodDirection getRandom(Random p_239631_0_) -// { -// return Util.getRandom(VALUES, p_239631_0_); -// } - -// public static LodDirection getNearest(double p_210769_0_, double p_210769_2_, double p_210769_4_) -// { -// return getNearest((float) p_210769_0_, (float) p_210769_2_, (float) p_210769_4_); -// } - -// public static LodDirection getNearest(float p_176737_0_, float p_176737_1_, float p_176737_2_) -// { -// LodDirection lodDirection = NORTH; -// float f = Float.MIN_VALUE; -// -// for (LodDirection direction1 : VALUES) -// { -// float f1 = p_176737_0_ * direction1.normal.x + p_176737_1_ * direction1.normal.y + p_176737_2_ * direction1.normal.z; -// if (f1 > f) -// { -// f = f1; -// lodDirection = direction1; -// } -// } -// -// return lodDirection; -// } - public static EDhDirection get(EDhDirection.AxisDirection p_181076_0_, EDhDirection.Axis p_181076_1_) - { - for (EDhDirection lodDirection : VALUES) - { - if (lodDirection.getAxisDirection() == p_181076_0_ && lodDirection.getAxis() == p_181076_1_) - { - return lodDirection; - } - } - - throw new IllegalArgumentException("No such direction: " + p_181076_0_ + " " + p_181076_1_); - } + //================// + // helper classes // + //================// - public Vec3i getNormal() + /** + * X
+ * Y
+ * Z
+ */ + public enum Axis { - return this.normal; - } - -// public boolean isFacingAngle(float p_243532_1_) -// { -// float f = p_243532_1_ * ((float) Math.PI / 180F); -// float f1 = -MathHelper.sin(f); -// float f2 = MathHelper.cos(f); -// return this.normal.getX() * f1 + this.normal.getZ() * f2 > 0.0F; -// } - - public enum Axis implements Predicate - { - X("x") - { - @Override - public int choose(int x, int y, int z) - { - return x; - } - - @Override - public double choose(double x, double y, double z) - { - return x; - } - }, - Y("y") - { - @Override - public int choose(int x, int y, int z) - { - return y; - } - - @Override - public double choose(double x, double y, double z) - { - return y; - } - }, - Z("z") - { - @Override - public int choose(int x, int y, int z) - { - return z; - } - - @Override - public double choose(double x, double y, double z) - { - return z; - } - }; + X("x"), + Y("y"), + Z("z"); - private static final EDhDirection.Axis[] VALUES = values(); + public final String name; - private static final Map BY_NAME = Arrays.stream(VALUES).collect(Collectors.toMap(EDhDirection.Axis::getName, (p_199785_0_) -> - { - return p_199785_0_; - })); - private final String name; - Axis(String name) - { - this.name = name; - } + //=============// + // constructor // + //=============// - public static EDhDirection.Axis byName(String name) - { - return BY_NAME.get(name.toLowerCase(Locale.ROOT)); - } + Axis(String name) { this.name = name; } - public String getName() - { - return this.name; - } - public boolean isVertical() - { - return this == Y; - } - public boolean isHorizontal() - { - return this == X || this == Z; - } + //=========// + // methods // + //=========// + + public boolean isVertical() { return this == Y; } + public boolean isHorizontal() { return this == X || this == Z; } @Override - public String toString() - { - return this.name; - } - -// public static LodDirection.Axis getRandom(Random p_239634_0_) -// { -// return Util.getRandom(VALUES, p_239634_0_); -// } + public String toString() { return this.name; } - @Override - public boolean test(EDhDirection p_test_1_) - { - return p_test_1_ != null && p_test_1_.getAxis() == this; - } - -// public LodDirection.Plane getPlane() -// { -// switch (this) -// { -// case X: -// case Z: -// return LodDirection.Plane.HORIZONTAL; -// case Y: -// return LodDirection.Plane.VERTICAL; -// default: -// throw new Error("Someone's been tampering with the universe!"); -// } -// } - - public abstract int choose(int p_196052_1_, int p_196052_2_, int p_196052_3_); - - public abstract double choose(double p_196051_1_, double p_196051_3_, double p_196051_5_); } + /** + * POSITIVE
+ * NEGATIVE
+ */ public enum AxisDirection { POSITIVE(1, "Towards positive"), NEGATIVE(-1, "Towards negative"); - private final int step; - private final String name; + public final int step; + public final String name; + + + + //=============// + // constructor // + //=============// AxisDirection(int newStep, String newName) { @@ -516,77 +182,20 @@ public enum EDhDirection this.name = newName; } - public int getStep() - { - return this.step; - } + + + //=========// + // methods // + //=========// + + public EDhDirection.AxisDirection opposite() + { return (this == POSITIVE) ? NEGATIVE : POSITIVE; } @Override - public String toString() - { - return this.name; - } + public String toString() { return this.name; } + + - public EDhDirection.AxisDirection opposite() - { - return this == POSITIVE ? NEGATIVE : POSITIVE; - } - } - -// public static enum Plane implements Iterable, Predicate -// { -// HORIZONTAL(new LodDirection[] { LodDirection.NORTH, LodDirection.EAST, LodDirection.SOUTH, LodDirection.WEST }, new LodDirection.Axis[] { LodDirection.Axis.X, LodDirection.Axis.Z }), -// VERTICAL(new LodDirection[] { LodDirection.UP, LodDirection.DOWN }, new LodDirection.Axis[] { LodDirection.Axis.Y }); -// -// private final LodDirection[] faces; -// private final LodDirection.Axis[] axis; -// -// private Plane(LodDirection[] p_i49393_3_, LodDirection.Axis[] p_i49393_4_) -// { -// this.faces = p_i49393_3_; -// this.axis = p_i49393_4_; -// } -// -// public LodDirection getRandomDirection(Random p_179518_1_) -// { -// return Util.getRandom(this.faces, p_179518_1_); -// } -// -// public LodDirection.Axis getRandomAxis(Random p_244803_1_) -// { -// return Util.getRandom(this.axis, p_244803_1_); -// } -// -// @Override -// public boolean test(@Nullable LodDirection p_test_1_) -// { -// return p_test_1_ != null && p_test_1_.getAxis().getPlane() == this; -// } -// -// @Override -// public Iterator iterator() -// { -// return Iterators.forArray(this.faces); -// } -// -// public Stream stream() -// { -// return Arrays.stream(this.faces); -// } -// } - - - - - public String getSerializedName() - { - return this.name; - } - - @Override - public String toString() - { - return this.name; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java index 029519a38..8f61356b9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java @@ -314,7 +314,7 @@ public class DhLightingEngine // propagate the lighting in each cardinal direction, IE: -x, +x, -y, +y, -z, +z - for (EDhDirection direction : EDhDirection.CARDINAL_DIRECTIONS) // since this is an array instead of an ArrayList this advanced for-loop shouldn't cause any GC issues + for (EDhDirection direction : EDhDirection.ALL) // since this is an array instead of an ArrayList this advanced for-loop shouldn't cause any GC issues { lightPos.mutateOffset(direction, neighbourBlockPos); neighbourBlockPos.mutateToChunkRelativePos(relNeighbourBlockPos); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index bf38a563e..7b0315396 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -344,8 +344,8 @@ public class DhSectionPos } return DhSectionPos.encode(getDetailLevel(pos), - getX(pos) + dir.getNormal().x, - getZ(pos) + dir.getNormal().z); + getX(pos) + dir.normal.x, + getZ(pos) + dir.normal.z); } @Deprecated diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPos.java index 61e81a192..0ae896d92 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPos.java @@ -75,9 +75,9 @@ public class DhBlockPos implements INetworkObject //========// /** creates a new {@link DhBlockPos} with the given offset from the current pos. */ - public DhBlockPos createOffset(EDhDirection direction) { return this.mutateOrCreateOffset(direction.getNormal().x, direction.getNormal().y, direction.getNormal().z, null); } + public DhBlockPos createOffset(EDhDirection direction) { return this.mutateOrCreateOffset(direction.normal.x, direction.normal.y, direction.normal.z, null); } /** if not null, mutates "mutablePos" so it matches the current pos after being offset. Otherwise creates a new {@link DhBlockPos}. */ - public void mutateOffset(EDhDirection direction, @NotNull DhBlockPosMutable mutablePos) { this.mutateOrCreateOffset(direction.getNormal().x, direction.getNormal().y, direction.getNormal().z, mutablePos); } + public void mutateOffset(EDhDirection direction, @NotNull DhBlockPosMutable mutablePos) { this.mutateOrCreateOffset(direction.normal.x, direction.normal.y, direction.normal.z, mutablePos); } public DhBlockPos createOffset(int x, int y, int z) { return this.mutateOrCreateOffset(x,y,z, null); } public void mutateOffset(int x, int y, int z, @NotNull DhBlockPosMutable mutablePos) { this.mutateOrCreateOffset(x, y, z, mutablePos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPosMutable.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPosMutable.java index 31b2eb6d2..d7d8f2763 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPosMutable.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/blockPos/DhBlockPosMutable.java @@ -50,7 +50,7 @@ public class DhBlockPosMutable extends DhBlockPos //========// /** @see DhBlockPos#createOffset(EDhDirection) */ - public DhBlockPosMutable createOffset(EDhDirection direction) { return new DhBlockPosMutable(super.mutateOrCreateOffset(direction.getNormal().x, direction.getNormal().y, direction.getNormal().z, null)); } + public DhBlockPosMutable createOffset(EDhDirection direction) { return new DhBlockPosMutable(super.mutateOrCreateOffset(direction.normal.x, direction.normal.y, direction.normal.z, null)); } /** @see DhBlockPos#createOffset(int, int, int) */ public DhBlockPosMutable createOffset(int x, int y, int z) { return new DhBlockPosMutable(this.mutateOrCreateOffset(x,y,z, null)); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 17d16598c..154a0cdb7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -640,7 +640,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen // clear cache // this.clearRenderCacheForPos(pos); - for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) + for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS) { long adjacentPos = DhSectionPos.getAdjacentPos(pos, direction); this.clearRenderCacheForPos(adjacentPos); @@ -658,7 +658,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen // the adjacent locations also need to be updated to make sure lighting // and water updates correctly, otherwise oceans may have walls // and lights may not show up over LOD borders - for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) + for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS) { long adjacentPos = DhSectionPos.getAdjacentPos(pos, direction); this.sectionsToReload.add(adjacentPos); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 3842b1ab2..aecf09bcc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -296,13 +296,13 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable CachedColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get(); CachedColumnRenderSource westRenderSource = adjacentLoadFutures[3].get()) { - ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; + ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.CARDINAL_COMPASS.length]; adjacentRenderSections[EDhDirection.NORTH.ordinal() - 2] = (northRenderSource != null) ? northRenderSource.columnRenderSource : null; adjacentRenderSections[EDhDirection.SOUTH.ordinal() - 2] = (southRenderSource != null) ? southRenderSource.columnRenderSource : null; adjacentRenderSections[EDhDirection.EAST.ordinal() - 2] = (eastRenderSource != null) ? eastRenderSource.columnRenderSource : null; adjacentRenderSections[EDhDirection.WEST.ordinal() - 2] = (westRenderSource != null) ? westRenderSource.columnRenderSource : null; - boolean[] adjIsSameDetailLevel = new boolean[EDhDirection.ADJ_DIRECTIONS.length]; + boolean[] adjIsSameDetailLevel = new boolean[EDhDirection.CARDINAL_COMPASS.length]; adjIsSameDetailLevel[EDhDirection.NORTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.NORTH); adjIsSameDetailLevel[EDhDirection.SOUTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.SOUTH); adjIsSameDetailLevel[EDhDirection.EAST.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.EAST); From 0567195f73d658b8a3c75c5034ecb46ec5e9c7e3 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 1 Nov 2025 16:27:54 -0400 Subject: [PATCH 03/24] minor datasource renaming --- .../fullData/sources/FullDataSourceV1.java | 4 ++-- .../fullData/sources/FullDataSourceV2.java | 3 +-- .../core/file/AbstractDataSourceHandler.java | 17 ++++++++--------- .../distanthorizons/core/file/IDataSource.java | 8 +++----- .../core/level/ClientLevelModule.java | 6 +----- 5 files changed, 15 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 3d2955dee..26306e728 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -48,7 +48,7 @@ import java.util.Arrays; * @see FullDataPointUtil * @see FullDataSourceV2 */ -public class FullDataSourceV1 implements IDataSource +public class FullDataSourceV1 implements IDataSource { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); @@ -103,7 +103,7 @@ public class FullDataSourceV1 implements IDataSource @Deprecated @Override - public boolean update(FullDataSourceV2 dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); } + public boolean update(FullDataSourceV2 dataSource) { throw new UnsupportedOperationException("Deprecated"); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 7838d941e..6a86c7027 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -61,7 +61,7 @@ import java.util.List; */ public class FullDataSourceV2 extends AbstractPhantomArrayList - implements IDataSource, IDhApiFullDataSource + implements IDataSource, IDhApiFullDataSource { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ @@ -375,7 +375,6 @@ public class FullDataSourceV2 //==========// @Override - public boolean update(@NotNull FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } public boolean update(@NotNull FullDataSourceV2 inputDataSource) { // don't try updating if the input is empty diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index fa88da0a3..109f396b5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -32,7 +32,7 @@ import java.util.concurrent.locks.ReentrantLock; * @see GeneratedFullDataSourceProvider */ public abstract class AbstractDataSourceHandler - , + , TRepo extends AbstractDhRepo, TDhLevel extends IDhLevel> @@ -74,7 +74,7 @@ public abstract class AbstractDataSourceHandler public final TRepo repo; - public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); + public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); @@ -92,7 +92,6 @@ public abstract class AbstractDataSourceHandler - //==================// // abstract methods // //==================// @@ -265,7 +264,7 @@ public abstract class AbstractDataSourceHandler { if (recipientDataSource != null) { - boolean dataModified = recipientDataSource.update(inputData, this.level); + boolean dataModified = recipientDataSource.update(inputData); if (dataModified) { // save the updated data to the database @@ -275,7 +274,7 @@ public abstract class AbstractDataSourceHandler } - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + for (IDataSourceUpdateListenerFunc listener : this.dateSourceUpdateListeners) { if (listener != null) { @@ -302,9 +301,9 @@ public abstract class AbstractDataSourceHandler - //================// - // helper methods // - //================// + //==================// + // debugger methods // + //==================// /** used for debugging to track which positions are queued for updating */ private void markUpdateStart(long dataSourcePos) @@ -368,7 +367,7 @@ public abstract class AbstractDataSourceHandler //================// @FunctionalInterface - public interface IDataSourceUpdateFunc + public interface IDataSourceUpdateListenerFunc { void OnDataSourceUpdated(TDataSource updatedFullDataSource); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index a8c014e70..afc973290 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -8,16 +8,14 @@ import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; /** * Base for all data sources.

* - * AutoCloseable Can be implemented to allow for disposing of pooled data sources.

- * - * @param there are times when we need specifically a client level vs a more generic level + * AutoCloseable Can be implemented to allow for disposing of pooled data sources. */ -public interface IDataSource extends IBaseDTO, AutoCloseable +public interface IDataSource extends IBaseDTO, AutoCloseable { long getPos(); /** @return true if the data was changed */ - boolean update(FullDataSourceV2 chunkData, TDhLevel level); + boolean update(FullDataSourceV2 chunkData); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 8ad98d26c..b73fd8038 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -19,8 +19,6 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; -import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; @@ -31,10 +29,8 @@ import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D; import com.seibel.distanthorizons.core.render.LodQuadTree; import com.seibel.distanthorizons.core.render.RenderBufferHandler; import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer; -import com.seibel.distanthorizons.core.render.renderer.LodRenderer; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; -import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import com.seibel.distanthorizons.core.logging.DhLogger; @@ -43,7 +39,7 @@ import java.io.Closeable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; -public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.IDataSourceUpdateFunc +public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.IDataSourceUpdateListenerFunc { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); From 47569f2b3cfd008c68627a761fc5512e9bd9be7f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 1 Nov 2025 16:33:07 -0400 Subject: [PATCH 04/24] minor dataSourceHandler refactor --- .../core/file/AbstractDataSourceHandler.java | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index 109f396b5..6554e0d06 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -143,45 +143,47 @@ public abstract class AbstractDataSourceHandler @Nullable public TDataSource get(long pos) { - TDataSource dataSource = null; try(TDTO dto = this.repo.getByKey(pos)) { - if (dto != null) + if (dto == null) { - try - { - // load from database - dataSource = this.createDataSourceFromDto(dto); - } - catch (DataCorruptedException e) - { - // there's a rare issue where the exception doesn't - // have a message, which can cause problems - String message = (e.getMessage() == null) ? e.getMessage() : "No Error message for exception ["+e.getClass().getSimpleName()+"]"; - - // Only log each message type once. - // This is done to prevent logging "No compression mode with the value [2]" 10,000 times - // if the user is migrating from a nightly build and used ZStd. - if (CORRUPT_DATA_ERRORS_LOGGED.add(message)) - { - LOGGER.warn("Corrupted data found at pos [" + DhSectionPos.toString(pos) + "]. Data at position will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + message + "].", e); - } - - this.repo.deleteWithKey(pos); - } + return this.makeEmptyDataSource(pos); } - else + + try { - dataSource = this.makeEmptyDataSource(pos); + // load from database + return this.createDataSourceFromDto(dto); + } + catch (DataCorruptedException e) + { + this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); + this.repo.deleteWithKey(pos); } } catch (InterruptedException ignore) { } catch (IOException e) { - LOGGER.warn("File read Error for pos ["+ DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); + LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); } - return dataSource; + // an error occurred + return null; + } + + protected void tryLogCorruptedDataError(String whereClause, Exception e) + { + // there's a rare issue where the exception doesn't + // have a message, which can cause problems + String message = (e.getMessage() == null) ? e.getMessage() : "No Error message for exception ["+e.getClass().getSimpleName()+"]"; + + // Only log each message type once. + // This is done to prevent logging "No compression mode with the value [2]" 10,000 times + // if the user is migrating from a nightly build and used ZStd. + if (CORRUPT_DATA_ERRORS_LOGGED.add(message)) + { + LOGGER.warn("Corrupted data found at [" + whereClause + "]. Data at will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + message + "].", e); + } } From bf05965015d47565581e6c30c0cab3637ccfbf75 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 2 Nov 2025 07:20:07 -0600 Subject: [PATCH 05/24] remove IDataSource --- .../methods/data/DhApiTerrainDataRepo.java | 3 +- .../fullData/sources/FullDataSourceV1.java | 29 +----------- .../fullData/sources/FullDataSourceV2.java | 44 ++++++------------- .../transformers/FullDataOcclusionCuller.java | 10 ++--- .../FullDataToRenderDataTransformer.java | 2 +- .../transformers/LodDataBuilder.java | 5 +-- .../core/file/IDataSource.java | 34 -------------- .../DelayedFullDataSourceSaveCache.java | 2 +- .../core/generation/DhLightingEngine.java | 4 +- .../core/sql/dto/FullDataSourceV2DTO.java | 1 + 10 files changed, 27 insertions(+), 107 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 25e271500..c27b8a58e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -49,7 +49,6 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.math.Vec3d; import com.seibel.distanthorizons.core.util.math.Vec3i; import it.unimi.dsi.fastutil.longs.LongArrayList; -import org.apache.logging.log4j.LogManager; import com.seibel.distanthorizons.core.logging.DhLogger; import org.jetbrains.annotations.Nullable; @@ -259,7 +258,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo //===============================// FullDataPointIdMap mapping = dataSource.mapping; - LongArrayList dataColumn = dataSource.get(relativePos.x, relativePos.z); + LongArrayList dataColumn = dataSource.getColumnAtRelPos(relativePos.x, relativePos.z); if (dataColumn != null) { int dataColumnIndexCount = dataColumn.size(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 26306e728..48d221ba2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -34,7 +33,6 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; -import com.seibel.distanthorizons.core.logging.DhLogger; import java.io.*; import java.util.Arrays; @@ -48,7 +46,7 @@ import java.util.Arrays; * @see FullDataPointUtil * @see FullDataSourceV2 */ -public class FullDataSourceV1 implements IDataSource +public class FullDataSourceV1 { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); @@ -95,28 +93,13 @@ public class FullDataSourceV1 implements IDataSource } - - - //======// - // data // - //======// - - @Deprecated - @Override - public boolean update(FullDataSourceV2 dataSource) { throw new UnsupportedOperationException("Deprecated"); } - - - //=====================// // setters and getters // //=====================// - @Override public Long getKey() { return this.pos; } - @Override public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); } - @Override public long getPos() { return this.pos; } public void resizeDataStructuresForRepopulation(long pos) @@ -125,7 +108,6 @@ public class FullDataSourceV1 implements IDataSource this.pos = pos; } - @Override public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - SECTION_SIZE_OFFSET); } public boolean isEmpty() { return this.isEmpty; } @@ -378,15 +360,6 @@ public class FullDataSourceV1 implements IDataSource public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } - //==================// - // override methods // - //==================// - - @Override - public void close() - { /* not currently needed */ } - - //================// // helper classes // diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 6a86c7027..4d3e1d8bf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -27,9 +27,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataOcclusionCuller; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; -import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; -import com.seibel.distanthorizons.core.file.IDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; @@ -45,7 +43,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; -import com.seibel.distanthorizons.core.logging.DhLogger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -61,7 +58,7 @@ import java.util.List; */ public class FullDataSourceV2 extends AbstractPhantomArrayList - implements IDataSource, IDhApiFullDataSource + implements IDhApiFullDataSource { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ @@ -87,10 +84,6 @@ public class FullDataSourceV2 private int cachedHashCode = 0; private final long pos; - @Override - public Long getKey() { return this.pos; } - @Override - public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); } public final FullDataPointIdMap mapping; @@ -226,7 +219,7 @@ public class FullDataSourceV2 private FullDataSourceV2( long pos, FullDataPointIdMap mapping, @Nullable LongArrayList[] data, - @Nullable byte[] columnGenerationSteps, @Nullable byte[] columnWorldCompressionMode, + byte @Nullable [] columnGenerationSteps, byte @Nullable [] columnWorldCompressionMode, boolean empty) { super(ARRAY_LIST_POOL, 2, 0, WIDTH * WIDTH); @@ -287,11 +280,11 @@ public class FullDataSourceV2 // getters // //=========// - public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException + public LongArrayList getColumnAtRelPos(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Nullable - public LongArrayList tryGet(int relX, int relZ) + public LongArrayList tryGetColumnAtRelPos(int relX, int relZ) { int index = tryGetRelativePosToIndex(relX, relZ); if (index == -1) @@ -306,7 +299,7 @@ public class FullDataSourceV2 * returns {@link FullDataPointUtil#EMPTY_DATA_POINT} if the given {@link DhBlockPos} * is outside this data source's boundaries. */ - public long getAtBlockPos(DhBlockPos blockPos) + public long getDataPointAtBlockPos(DhBlockPos blockPos) { DhLodPos requestedPos = new DhLodPos(LodUtil.BLOCK_DETAIL_LEVEL, blockPos.getX(), blockPos.getZ()); @@ -330,7 +323,7 @@ public class FullDataSourceV2 DhLodPos relativePos = requestedPos.getDhSectionRelativePositionForDetailLevel(requestDetailLevel); // get the data column - LongArrayList dataColumn = this.get(relativePos.x, relativePos.z); + LongArrayList dataColumn = this.getColumnAtRelPos(relativePos.x, relativePos.z); if (dataColumn == null) { return FullDataPointUtil.EMPTY_DATA_POINT; @@ -374,8 +367,7 @@ public class FullDataSourceV2 // updating // //==========// - @Override - public boolean update(@NotNull FullDataSourceV2 inputDataSource) + public boolean updateFromChunk(@NotNull FullDataSourceV2 inputDataSource) { // don't try updating if the input is empty if (inputDataSource.mapping.isEmpty()) @@ -403,7 +395,7 @@ public class FullDataSourceV2 // copy over application flag if either are set to continue propagating (BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent)) // don't propagate past the top of the tree - && (DhSectionPos.getDetailLevel(this.pos) < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL); } // null check to prevent setting a flag we don't want to save in the DB @@ -412,7 +404,7 @@ public class FullDataSourceV2 this.applyToChildren = (BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren)) // don't propagate past the bottom of the tree - && (DhSectionPos.getDetailLevel(this.pos) > AbstractDataSourceHandler.MIN_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.MIN_SECTION_DETAIL_LEVEL); } } else if (inputDetailLevel + 1 == thisDetailLevel) @@ -423,7 +415,7 @@ public class FullDataSourceV2 this.applyToParent = dataChanged && (BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent)) - && (DhSectionPos.getDetailLevel(this.pos) < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL); } else if (inputDetailLevel - 1 == thisDetailLevel) @@ -435,7 +427,7 @@ public class FullDataSourceV2 this.applyToChildren = dataChanged && (BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren)) - && (DhSectionPos.getDetailLevel(this.pos) > AbstractDataSourceHandler.MIN_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.MIN_SECTION_DETAIL_LEVEL); } else { @@ -460,7 +452,7 @@ public class FullDataSourceV2 { for (int z = 0; z < WIDTH; z++) { - LongArrayList dataColumn = this.get(x, z); + LongArrayList dataColumn = this.getColumnAtRelPos(x, z); if (dataColumn != null && dataColumn.size() > 1) { @@ -1202,18 +1194,10 @@ public class FullDataSourceV2 // setters and getters // //=====================// - @Override public long getPos() { return this.pos; } - @Override public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); } - public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) - { - int index = relativePosToIndex(relX, relZ); - return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps.getByte(index)); - } - public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep, EDhApiWorldCompressionMode worldCompressionMode) { int index = relativePosToIndex(relX, relZ); @@ -1277,7 +1261,7 @@ public class FullDataSourceV2 @Override public List getApiDataPointColumn(int relX, int relZ) throws IndexOutOfBoundsException { - LongArrayList dataColumn = this.get(relX, relZ); + LongArrayList dataColumn = this.getColumnAtRelPos(relX, relZ); ArrayList apiList = new ArrayList<>(); for (int i = 0; i < dataColumn.size(); i++) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataOcclusionCuller.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataOcclusionCuller.java index ec776a62f..67d37f225 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataOcclusionCuller.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataOcclusionCuller.java @@ -22,11 +22,11 @@ public class FullDataOcclusionCuller int relX, int relZ ) { - LongArrayList centerColumn = dataSource.get(relX, relZ); - LongArrayList posXColumn = dataSource.tryGet(relX + 1, relZ); - LongArrayList negXColumn = dataSource.tryGet(relX - 1, relZ); - LongArrayList posZColumn = dataSource.tryGet(relX, relZ + 1); - LongArrayList negZColumn = dataSource.tryGet(relX, relZ - 1); + LongArrayList centerColumn = dataSource.getColumnAtRelPos(relX, relZ); + LongArrayList posXColumn = dataSource.tryGetColumnAtRelPos(relX + 1, relZ); + LongArrayList negXColumn = dataSource.tryGetColumnAtRelPos(relX - 1, relZ); + LongArrayList posZColumn = dataSource.tryGetColumnAtRelPos(relX, relZ + 1); + LongArrayList negZColumn = dataSource.tryGetColumnAtRelPos(relX, relZ - 1); if (posXColumn == null || posXColumn.size() == 0 || negXColumn == null || negXColumn.size() == 0 diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 4330a1da9..5d43cea48 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -126,7 +126,7 @@ public class FullDataToRenderDataTransformer for (int z = 0; z < FullDataSourceV2.WIDTH; z++) { ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - LongArrayList dataColumn = fullDataSource.get(x, z); + LongArrayList dataColumn = fullDataSource.getColumnAtRelPos(x, z); updateOrReplaceRenderDataViewColumnWithFullDataColumn( levelWrapper, fullDataSource, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index d454fbf20..dbf6f46d2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -24,8 +24,6 @@ import java.util.List; import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.api.interfaces.block.IDhApiBiomeWrapper; -import com.seibel.distanthorizons.api.interfaces.block.IDhApiBlockStateWrapper; import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiChunkProcessingEvent; import com.seibel.distanthorizons.api.objects.data.DhApiChunk; import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint; @@ -47,7 +45,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector; import it.unimi.dsi.fastutil.longs.LongArrayList; import com.seibel.distanthorizons.core.logging.DhLogger; -import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; public class LodDataBuilder @@ -142,7 +139,7 @@ public class LodDataBuilder int columnZ = relBlockZ + chunkOffsetZ; // Get column data - LongArrayList longs = dataSource.get(columnX, columnZ); + LongArrayList longs = dataSource.getColumnAtRelPos(columnX, columnZ); if (longs == null) { longs = new LongArrayList(dataCapacity); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java deleted file mode 100644 index afc973290..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.seibel.distanthorizons.core.file; - -import com.seibel.distanthorizons.api.enums.EDhApiDetailLevel; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; - -/** - * Base for all data sources.

- * - * AutoCloseable Can be implemented to allow for disposing of pooled data sources. - */ -public interface IDataSource extends IBaseDTO, AutoCloseable -{ - long getPos(); - - /** @return true if the data was changed */ - boolean update(FullDataSourceV2 chunkData); - - - - //===========// - // meta data // - //===========// - - /** - * Returns the detail level of the data contained by this data source. - * IE: 0 for block, 1 for 2x2 blocks, etc. - * - * @see EDhApiDetailLevel - */ - byte getDataDetailLevel(); - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java index 94d1d2dbe..8ca11ef03 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java @@ -104,7 +104,7 @@ public class DelayedFullDataSourceSaveCache implements AutoCloseable } // write the new data into memory - memoryDataSource.update(inputDataSource); + memoryDataSource.updateFromChunk(inputDataSource); // keep track of when the last time we saved something was pair.updateLastWrittenTimestamp(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java index 8f61356b9..2e1632acb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java @@ -413,7 +413,7 @@ public class DhLightingEngine { for (int x = 0; x < FullDataSourceV2.WIDTH; x++) { - LongArrayList dataPoints = dataSource.get(x, z); + LongArrayList dataPoints = dataSource.getColumnAtRelPos(x, z); if (dataPoints != null && !dataPoints.isEmpty()) { // iterate through the data points in this column top-down @@ -564,7 +564,7 @@ public class DhLightingEngine // check if the adjacent position is within the bounds of this data source... if (adjacentX >= 0 && adjacentX < FullDataSourceV2.WIDTH && adjacentZ >= 0 && adjacentZ < FullDataSourceV2.WIDTH) { - LongArrayList adjacentDataPoints = chunk.get(adjacentX, adjacentZ); + LongArrayList adjacentDataPoints = chunk.getColumnAtRelPos(adjacentX, adjacentZ); // ...and also check to make sure we have some data points // (potentially transparent ones) to propagate through in the adjacent column. if (adjacentDataPoints != null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 5a82322bc..d49d40ea9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -121,6 +121,7 @@ public class FullDataSourceV2DTO /** Should only be used for subsequent decoding */ public static FullDataSourceV2DTO CreateEmptyDataSourceForDecoding() { return new FullDataSourceV2DTO(); } + private FullDataSourceV2DTO() { super(ARRAY_LIST_POOL, 4, 0, 0); From 4d4d8fd8e9570debe6e3cedbddf658a2f5951145 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 4 Nov 2025 07:46:06 -0600 Subject: [PATCH 06/24] Split up full data source provider into multiple classes --- .../fullData/sources/FullDataSourceV2.java | 2 +- .../core/file/AbstractDataSourceHandler.java | 377 -------- .../FullDataSourceProviderV2.java | 814 ------------------ .../GeneratedFullDataSourceProvider.java | 31 +- .../IDataSourceUpdateListenerFunc.java | 7 + .../{ => V1}/FullDataSourceProviderV1.java | 6 +- .../file/fullDatafile/V2/DataMigratorV1.java | 346 ++++++++ .../V2/FullDataSourceProviderV2.java | 396 +++++++++ .../V2/FullDataUpdatePropagatorV2.java | 408 +++++++++ .../fullDatafile/V2/FullDataUpdaterV2.java | 216 +++++ .../core/level/AbstractDhServerLevel.java | 25 +- .../core/level/ClientLevelModule.java | 13 +- .../core/level/DhClientLevel.java | 33 +- .../distanthorizons/core/level/IDhLevel.java | 2 +- .../core/render/LodQuadTree.java | 2 +- .../core/render/LodRenderSection.java | 2 +- 16 files changed, 1412 insertions(+), 1268 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IDataSourceUpdateListenerFunc.java rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{ => V1}/FullDataSourceProviderV1.java (95%) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/DataMigratorV1.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 4d3e1d8bf..eebb24e92 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -27,7 +27,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataOcclusionCuller; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java deleted file mode 100644 index 6554e0d06..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ /dev/null @@ -1,377 +0,0 @@ -package com.seibel.distanthorizons.core.file; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.structure.ISaveStructure; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLogger; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; -import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; -import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider; -import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; -import com.seibel.distanthorizons.core.logging.DhLogger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -/** - * @see FullDataSourceProviderV2 - * @see RemoteFullDataSourceProvider - * @see GeneratedFullDataSourceProvider - */ -public abstract class AbstractDataSourceHandler - , - TRepo extends AbstractDhRepo, - TDhLevel extends IDhLevel> - implements AutoCloseable -{ - private static final DhLogger LOGGER = new DhLoggerBuilder().build(); - private static final Set CORRUPT_DATA_ERRORS_LOGGED = Collections.newSetFromMap(new ConcurrentHashMap<>()); - - - /** - * The highest numerical detail level possible. - * Used when determining which positions to update. - * - * @see AbstractDataSourceHandler#MIN_SECTION_DETAIL_LEVEL - */ - public static final byte TOP_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + LodUtil.REGION_DETAIL_LEVEL; - /** - * The lowest numerical detail level possible. - * - * @see AbstractDataSourceHandler#TOP_SECTION_DETAIL_LEVEL - */ - public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - - - protected final PositionalLockProvider updateLockProvider = new PositionalLockProvider(); - /** - * generally just used for debugging, - * keeps track of which positions are currently locked. - */ - public final Set lockedPosSet = ConcurrentHashMap.newKeySet(); - public final ConcurrentHashMap queuedUpdateCountsByPos = new ConcurrentHashMap<>(); - - - protected final ReentrantLock closeLock = new ReentrantLock(); - protected volatile boolean isShutdown = false; - - protected final TDhLevel level; - protected final File saveDir; - - public final TRepo repo; - - public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); - - - - //=============// - // constructor // - //=============// - - public AbstractDataSourceHandler(TDhLevel level, ISaveStructure saveStructure) { this(level, saveStructure, null); } - public AbstractDataSourceHandler(TDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) - { - this.level = level; - this.saveDir = (saveDirOverride == null) ? saveStructure.getSaveFolder(level.getLevelWrapper()) : saveDirOverride; - this.repo = this.createRepo(); - } - - - - //==================// - // abstract methods // - //==================// - - /** When this is called the parent folders should be created */ - protected abstract TRepo createRepo(); - - protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException, DataCorruptedException; - protected abstract TDTO createDtoFromDataSource(TDataSource dataSource); - - protected abstract TDataSource makeEmptyDataSource(long pos); - - - - //==============// - // data reading // - //==============// - - /** - * Returns the {@link TDataSource} for the given section position.
- * The returned data source may be null if repo is in the process of shutting down.

- * - * This call is concurrent. I.e. it supports being called by multiple threads at the same time. - */ - public CompletableFuture getAsync(long pos) - { - AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return CompletableFuture.completedFuture(null); - } - - - try - { - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); - } - catch (RejectedExecutionException ignore) - { - // the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back - return CompletableFuture.completedFuture(null); - } - } - /** - * Should only be used in internal file handler methods where we are already running on a file handler thread. - * Can return null if the repo is in the process of being shut down - * @see AbstractDataSourceHandler#getAsync(long) - */ - @Nullable - public TDataSource get(long pos) - { - try(TDTO dto = this.repo.getByKey(pos)) - { - if (dto == null) - { - return this.makeEmptyDataSource(pos); - } - - try - { - // load from database - return this.createDataSourceFromDto(dto); - } - catch (DataCorruptedException e) - { - this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); - this.repo.deleteWithKey(pos); - } - } - catch (InterruptedException ignore) { } - catch (IOException e) - { - LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); - } - - // an error occurred - return null; - } - - protected void tryLogCorruptedDataError(String whereClause, Exception e) - { - // there's a rare issue where the exception doesn't - // have a message, which can cause problems - String message = (e.getMessage() == null) ? e.getMessage() : "No Error message for exception ["+e.getClass().getSimpleName()+"]"; - - // Only log each message type once. - // This is done to prevent logging "No compression mode with the value [2]" 10,000 times - // if the user is migrating from a nightly build and used ZStd. - if (CORRUPT_DATA_ERRORS_LOGGED.add(message)) - { - LOGGER.warn("Corrupted data found at [" + whereClause + "]. Data at will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + message + "].", e); - } - } - - - - //===============// - // data updating // - //===============// - - /** - * Can be used if the same thread is already handling IO and/or LOD generation. - * Otherwise the async version {@link AbstractDataSourceHandler#updateDataSourceAsync(FullDataSourceV2)} may be a better choice. - */ - public void updateDataSource(@NotNull FullDataSourceV2 inputDataSource) - { this.updateDataSourceAtPos(inputDataSource.getPos(), inputDataSource, true); } - - /** - * Can be used if you don't want to lock the current thread - * Otherwise the sync version {@link AbstractDataSourceHandler#updateDataSource(FullDataSourceV2)} may be a better choice. - */ - public CompletableFuture updateDataSourceAsync(@NotNull FullDataSourceV2 inputDataSource) - { - AbstractExecutorService executor = ThreadPoolUtil.getChunkToLodBuilderExecutor(); - if (executor == null || executor.isTerminated()) - { - return CompletableFuture.completedFuture(null); - } - - - try - { - this.markUpdateStart(inputDataSource.getPos()); - return CompletableFuture.runAsync(() -> - { - try - { - this.updateDataSourceAtPos(inputDataSource.getPos(), inputDataSource, true); - } - catch (Exception e) - { - LOGGER.error("Unexpected error in async data source update at pos: ["+DhSectionPos.toString(inputDataSource.getPos())+"], error: ["+e.getMessage()+"].", e); - } - finally - { - this.markUpdateEnd(inputDataSource.getPos()); - } - }, executor); - } - catch (RejectedExecutionException ignore) - { - // can happen if the executor was shutdown while this task was queued - this.markUpdateEnd(inputDataSource.getPos()); - return CompletableFuture.completedFuture(null); - } - } - - /** - * After this method returns the inputData will be written to file. - * - * @param updatePos the position to update - */ - protected void updateDataSourceAtPos(long updatePos, @NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos) - { - boolean methodLocked = false; - // a lock is necessary to prevent two threads from writing to the same position at once, - // if that happens only the second update will apply and the LOD will end up with hole(s) - ReentrantLock updateLock = this.updateLockProvider.getLock(updatePos); - - try - { - if (lockOnUpdatePos) - { - methodLocked = true; - updateLock.lock(); - this.lockedPosSet.add(updatePos); - } - - - // get or create the data source - try (TDataSource recipientDataSource = this.get(updatePos)) - { - if (recipientDataSource != null) - { - boolean dataModified = recipientDataSource.update(inputData); - if (dataModified) - { - // save the updated data to the database - try (TDTO dto = this.createDtoFromDataSource(recipientDataSource)) - { - this.repo.save(dto); - } - - - for (IDataSourceUpdateListenerFunc listener : this.dateSourceUpdateListeners) - { - if (listener != null) - { - listener.OnDataSourceUpdated(recipientDataSource); - } - } - } - } - } - } - catch (Exception e) - { - LOGGER.error("Error updating pos ["+DhSectionPos.toString(updatePos)+"], error: "+e.getMessage(), e); - } - finally - { - if (methodLocked) - { - updateLock.unlock(); - this.lockedPosSet.remove(updatePos); - } - } - } - - - - //==================// - // debugger methods // - //==================// - - /** used for debugging to track which positions are queued for updating */ - private void markUpdateStart(long dataSourcePos) - { - this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) -> - { - if (atomicCount == null) - { - atomicCount = new AtomicInteger(0); - } - atomicCount.incrementAndGet(); - return atomicCount; - }); - } - /** used for debugging to track which positions are queued for updating */ - private void markUpdateEnd(long dataSourcePos) - { - this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) -> - { - if (atomicCount != null && atomicCount.decrementAndGet() <= 0) - { - atomicCount = null; - } - return atomicCount; - }); - } - - - - //=========// - // cleanup // - //=========// - - @Override - public void close() - { - try - { - this.closeLock.lock(); - this.isShutdown = true; - - // wait a moment so any queued saves can finish queuing, - // otherwise we might not see everything that needs saving and attempt to use a closed repo - Thread.sleep(200); - - LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "]."); - - this.repo.close(); - } - catch (InterruptedException ignore) { } - finally - { - this.closeLock.unlock(); - } - } - - - - //================// - // helper classes // - //================// - - @FunctionalInterface - public interface IDataSourceUpdateListenerFunc - { - void OnDataSourceUpdated(TDataSource updatedFullDataSource); - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java deleted file mode 100644 index 0459896e2..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ /dev/null @@ -1,814 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.fullDatafile; - -import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; -import com.seibel.distanthorizons.core.api.internal.ClientApi; -import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.file.structure.ISaveStructure; -import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; -import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLogger; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.logging.f3.F3Screen; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos; -import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; -import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; -import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; -import com.seibel.distanthorizons.core.util.ThreadUtil; -import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; -import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker; -import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; -import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; -import it.unimi.dsi.fastutil.longs.LongArrayList; -import com.seibel.distanthorizons.core.logging.DhLogger; -import org.jetbrains.annotations.Nullable; - -import java.awt.*; -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; -import java.text.NumberFormat; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Handles reading/writing {@link FullDataSourceV2} - * to and from the database. - */ -public class FullDataSourceProviderV2 - extends AbstractDataSourceHandler - implements IDebugRenderable -{ - private static final DhLogger LOGGER = new DhLoggerBuilder().build(); - private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); - - protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 5; - /** how many parent update tasks can be in the queue at once */ - protected static int getMaxUpdateTaskCount() { return NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD* Config.Common.MultiThreading.numberOfThreads.get(); } - - /** indicates how long the update queue thread should wait between queuing ticks */ - protected static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250; - - /** how many data sources should be pulled down for migration at once */ - private static final int MIGRATION_BATCH_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD; - /** - * 5 minutes
- * This should be much longer than any update should take. This is just - * to make sure the thread doesn't get stuck. - */ - private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000; - - - - /** - * Interrupting the migration thread pool doesn't work well and may corrupt the database - * vs gracefully shutting down the thread ourselves. - */ - protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); - protected final FullDataSourceProviderV1 legacyFileHandler; - - protected boolean migrationStartMessageQueued = false; - - protected long legacyDeletionCount = -1; - protected long migrationCount = -1; - protected boolean migrationStoppedWithError = false; - - /** - * Tracks which positions are currently being updated - * to prevent duplicate concurrent updates. - */ - public final Set updatingPosSet = ConcurrentHashMap.newKeySet(); - - // TODO only run thread if modifications happened recently - /** - * This isn't in {@link AbstractDataSourceHandler} since we only want to update - * the newest version of the full data, so if we have providers for either - * render data or old full data, we don't want to update them.

- * - * Will be null on the dedicated server since updates don't need to be propagated, - * only the highest detail level is needed. - */ - @Nullable - private final ThreadPoolExecutor updateQueueProcessor; - - - - //=============// - // constructor // - //=============// - - public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure) { this(level, saveStructure, null); } - public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) - { - super(level, saveStructure, saveDirOverride); - this.legacyFileHandler = new FullDataSourceProviderV1<>(level, saveStructure, saveDirOverride); - - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus); - - String levelId = level.getLevelWrapper().getDhIdentifier(); - - // start migrating any legacy data sources present in the background - ThreadPoolExecutor executor = ThreadPoolUtil.getFullDataMigrationExecutor(); - if (executor != null) - { - executor.execute(this::convertLegacyDataSources); - } - else - { - // shouldn't happen, but just in case - LOGGER.error("Unable to start migration for level: ["+levelId+"] due to missing executor."); - } - - // update propagation doesn't need to be run on the server since only the highest detail level is needed - this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Parent Update Queue [" + levelId + "]"); - this.updateQueueProcessor.execute(this::runUpdateQueue); - } - - - - //====================// - // Abstract overrides // - //====================// - - @Override - protected FullDataSourceV2Repo createRepo() - { - try - { - return new FullDataSourceV2Repo(AbstractDhRepo.DEFAULT_DATABASE_TYPE, new File(this.saveDir.getPath() + File.separator + ISaveStructure.DATABASE_NAME)); - } - catch (SQLException e) - { - // should only happen if there is an issue with the database (it's locked or the folder path is missing) - // or the database update failed - throw new RuntimeException(e); - } - } - - @Override - protected FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource) - { - try - { - // when creating new data use the compressor currently selected in the config - EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get(); - return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum); - } - catch (IOException e) - { - LOGGER.warn("Unable to create DTO, error: "+e.getMessage(), e); - return null; - } - } - - @Override - protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException - { return dto.createDataSource(this.level.getLevelWrapper()); } - - @Override - protected FullDataSourceV2 makeEmptyDataSource(long pos) - { return FullDataSourceV2.createEmpty(pos); } - - - - //================// - // parent updates // - //================// - - private void runUpdateQueue() - { - while (!Thread.interrupted()) - { - try - { - Thread.sleep(UPDATE_QUEUE_THREAD_DELAY_IN_MS); - - PriorityTaskPicker.Executor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); - if (executor == null || executor.isTerminated()) - { - continue; - } - - // TODO it might be worth skipping this logic if no parent updates happened - - // update positions closest to the player (if not on a server) - // to make world gen appear faster - DhBlockPos targetBlockPos = DhBlockPos.ZERO; - if (MC_CLIENT != null && MC_CLIENT.playerExists()) - { - targetBlockPos = MC_CLIENT.getPlayerBlockPos(); - } - - this.runParentUpdates(executor, targetBlockPos); - - if (Config.Common.LodBuilding.Experimental.upsampleLowerDetailLodsToFillHoles.get()) - { - this.runChildUpdates(executor, targetBlockPos); - } - - } - catch (InterruptedException ignored) - { - Thread.currentThread().interrupt(); - } - catch (Exception e) - { - LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e); - } - } - - LOGGER.info("Update thread ["+Thread.currentThread().getName()+"] terminated."); - } - /** will always apply updates */ - private void runParentUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos) - { - int maxUpdateTaskCount = getMaxUpdateTaskCount(); - - // queue parent updates - if (executor.getQueueSize() < maxUpdateTaskCount - && this.updatingPosSet.size() < maxUpdateTaskCount) - { - // get the positions that need to be applied to their parents - LongArrayList parentUpdatePosList = this.repo.getPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount); - - // combine updates together based on their parent - HashMap> updatePosByParentPos = new HashMap<>(); - for (Long pos : parentUpdatePosList) - { - updatePosByParentPos.compute(DhSectionPos.getParentPos(pos), (parentPos, updatePosSet) -> - { - if (updatePosSet == null) - { - updatePosSet = new HashSet<>(); - } - updatePosSet.add(pos); - return updatePosSet; - }); - } - - // queue the updates - for (Long parentUpdatePos : updatePosByParentPos.keySet()) - { - // stop if there are already a bunch of updates queued - if (this.updatingPosSet.size() > maxUpdateTaskCount - || executor.getQueueSize() > maxUpdateTaskCount - || !this.updatingPosSet.add(parentUpdatePos)) - { - break; - } - - try - { - executor.execute(() -> - { - ReentrantLock parentWriteLock = this.updateLockProvider.getLock(parentUpdatePos); - boolean parentLocked = false; - try - { - //LOGGER.info("updating parent: "+parentUpdatePos); - - // Locking the parent before the children should prevent deadlocks. - // TryLock is used instead of lock so this thread can handle a different update. - if (parentWriteLock.tryLock()) - { - parentLocked = true; - this.lockedPosSet.add(parentUpdatePos); - - try (FullDataSourceV2 parentDataSource = this.get(parentUpdatePos)) - { - // will return null if the file handler is shutting down - if (parentDataSource != null) - { - // apply each child pos to the parent - for (Long childPos : updatePosByParentPos.get(parentUpdatePos)) - { - ReentrantLock childReadLock = this.updateLockProvider.getLock(childPos); - try - { - childReadLock.lock(); - this.lockedPosSet.add(childPos); - - try (FullDataSourceV2 childDataSource = this.get(childPos)) - { - // can return null when the file handler is being shut down - if (childDataSource != null) - { - parentDataSource.update(childDataSource); - } - } - } - catch (Exception e) - { - LOGGER.error("Unexpected in parent update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e); - } - finally - { - childReadLock.unlock(); - this.lockedPosSet.remove(childPos); - } - } - - - if (DhSectionPos.getDetailLevel(parentUpdatePos) < TOP_SECTION_DETAIL_LEVEL) - { - parentDataSource.applyToParent = true; - } - - this.updateDataSourceAtPos(parentUpdatePos, parentDataSource, false); - for (Long childPos : updatePosByParentPos.get(parentUpdatePos)) - { - this.repo.setApplyToParent(childPos, false); - } - } - } - } - } - finally - { - if (parentLocked) - { - parentWriteLock.unlock(); - this.lockedPosSet.remove(parentUpdatePos); - } - - this.updatingPosSet.remove(parentUpdatePos); - } - }); - } - catch (RejectedExecutionException ignore) - { /* the executor was shut down, it should be back up shortly and able to accept new jobs */ } - catch (Exception e) - { - this.updatingPosSet.remove(parentUpdatePos); - throw e; - } - } - } - } - /** stops if it finds any LOD data */ - private void runChildUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos) - { - int maxUpdateTaskCount = getMaxUpdateTaskCount(); - - // queue child updates - if (executor.getQueueSize() < maxUpdateTaskCount - && this.updatingPosSet.size() < maxUpdateTaskCount) - { - // get the positions that need to be applied to their children - LongArrayList childUpdatePosList = this.repo.getChildPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount); - - // queue the updates - for (long parentUpdatePos : childUpdatePosList) - { - // stop if there are already a bunch of updates queued - if (this.updatingPosSet.size() > maxUpdateTaskCount - || executor.getQueueSize() > maxUpdateTaskCount) - { - break; - } - - // skip already updating positions - if (!this.updatingPosSet.add(parentUpdatePos)) - { - continue; - } - - try - { - executor.execute(() -> - { - ReentrantLock parentReadLock = this.updateLockProvider.getLock(parentUpdatePos); - boolean parentLocked = false; - try - { - //LOGGER.info("updating parent: "+parentUpdatePos); - - // Locking the parent before the children should prevent deadlocks. - // TryLock is used instead of lock so this thread can handle a different update. - if (parentReadLock.tryLock()) - { - parentLocked = true; - this.lockedPosSet.add(parentUpdatePos); - - try (FullDataSourceV2 parentDataSource = this.get(parentUpdatePos)) - { - // will return null if the file handler is shutting down - if (parentDataSource != null) - { - // apply parent to each child - for (int i = 0; i < 4; i++) - { - long childPos = DhSectionPos.getChildByIndex(parentUpdatePos, i); - - ReentrantLock childWriteLock = this.updateLockProvider.getLock(childPos); - try - { - childWriteLock.lock(); - this.lockedPosSet.add(childPos); - - try (FullDataSourceV2 childDataSource = this.get(childPos)) - { - // will return null if the file handler is shutting down - if (childDataSource != null) - { - childDataSource.update(parentDataSource); - - // don't propagate child updates past the bottom of the tree - if (DhSectionPos.getDetailLevel(childPos) != DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL) - { - childDataSource.applyToChildren = true; - } - - this.updateDataSourceAtPos(childPos, childDataSource, false); - } - } - } - catch (Exception e) - { - LOGGER.error("Unexpected in child update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e); - } - finally - { - childWriteLock.unlock(); - this.lockedPosSet.remove(childPos); - } - } - - this.repo.setApplyToChild(parentUpdatePos, false); - } - } - } - } - finally - { - if (parentLocked) - { - parentReadLock.unlock(); - this.lockedPosSet.remove(parentUpdatePos); - } - - this.updatingPosSet.remove(parentUpdatePos); - } - }); - } - catch (RejectedExecutionException ignore) - { /* the executor was shut down, it should be back up shortly and able to accept new jobs */ } - catch (Exception e) - { - this.updatingPosSet.remove(parentUpdatePos); - throw e; - } - } - } - } - - - - //=======================// - // data source migration // - //=======================// - - private void convertLegacyDataSources() - { - try - { - String levelId = this.level.getLevelWrapper().getDhIdentifier(); - LOGGER.info("Attempting to migrate data sources for: [" + levelId + "]-[" + this.saveDir + "]..."); - this.migrationThreadRunning.set(true); - - - - //============================// - // delete unused data sources // - //============================// - - // this could be done all at once via SQL, - // but doing it in chunks prevents locking the database for long periods of time - long unusedCount = 0; - long totalDeleteCount = this.legacyFileHandler.repo.getUnusedDataSourceCount(); - if (totalDeleteCount != 0) - { - // this should only be shown once per session but should be shown during - // either when the deletion or migration phases start - this.showMigrationStartMessage(); - - - LOGGER.info("deleting [" + levelId + "] - [" + totalDeleteCount + "] unused data sources..."); - this.legacyDeletionCount = totalDeleteCount; - - ArrayList unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50); - while (unusedDataPosList.size() != 0) - { - unusedCount += unusedDataPosList.size(); - this.legacyDeletionCount -= unusedDataPosList.size(); - - - long startTime = System.currentTimeMillis(); - - // delete batch and get next batch - this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList); - unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50); - - long endStart = System.currentTimeMillis(); - long deleteTime = endStart - startTime; - LOGGER.info("Deleting [" + levelId + "] - [" + unusedCount + "/" + totalDeleteCount + "] in [" + deleteTime + "]ms ..."); - - - // a slight delay is added to prevent accidentally locking the database when deleting a lot of rows - // (that shouldn't be the case since we're using WAL journaling, but just in case) - try - { - // use the delete time so we don't make powerful computers wait super long - // and weak computers wait no time at all - Thread.sleep(deleteTime / 2); - } - catch (InterruptedException ignore) - { - } - } - LOGGER.info("Done deleting [" + levelId + "] - [" + totalDeleteCount + "] unused data sources."); - - } - - - - //===========// - // migration // - //===========// - - long totalMigrationCount = this.legacyFileHandler.getDataSourceMigrationCount(); - this.migrationCount = totalMigrationCount; - LOGGER.info("Found [" + totalMigrationCount + "] data sources that need migration."); - - ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); - if (!legacyDataSourceList.isEmpty()) - { - this.showMigrationStartMessage(); - - try - { - // keep going until every data source has been migrated - int progressCount = 0; - while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get()) - { - NumberFormat numFormat = F3Screen.NUMBER_FORMAT; - LOGGER.info("Migrating [" + levelId + "] - [" + numFormat.format(progressCount) + "/" + numFormat.format(totalMigrationCount) + "]..."); - - ArrayList> updateFutureList = new ArrayList<>(); - for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) - { - FullDataSourceV1 legacyDataSource = legacyDataSourceList.get(i); - - try - { - // convert the legacy data source to the new format, - // this is a relatively cheap operation - FullDataSourceV2 newDataSource = FullDataSourceV2.createFromLegacyDataSourceV1(legacyDataSource); - newDataSource.applyToParent = true; - - // the actual update process can be moderately expensive due to having to update - // the render data along with the full data, so running it async on the update threads gains us a good bit of speed - CompletableFuture future = this.updateDataSourceAsync(newDataSource); - updateFutureList.add(future); - future.thenRun(() -> - { - // after the update finishes the legacy data source can be safely deleted - this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos()); - newDataSource.close(); - }); - } - catch (Exception e) - { - Long migrationPos = legacyDataSource.getPos(); - LOGGER.warn("Unexpected issue migrating data source at pos [" + DhSectionPos.toString(migrationPos) + "]. Error: " + e.getMessage(), e); - this.legacyFileHandler.markMigrationFailed(migrationPos); - } - } - - - try - { - // wait for each thread to finish updating - CompletableFuture combinedFutures = CompletableFuture.allOf(updateFutureList.toArray(new CompletableFuture[0])); - combinedFutures.get(MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS); - } - catch (InterruptedException | TimeoutException e) - { - LOGGER.warn("Migration update timed out after [" + MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS + "] milliseconds. Migration will re-try the same positions again in a moment.", e); - } - catch (ExecutionException e) - { - LOGGER.warn("Migration update failed. Migration will re-try the same positions again. Error:" + e.getMessage(), e); - } - - legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); - - progressCount += legacyDataSourceList.size(); - this.migrationCount -= legacyDataSourceList.size(); - } - } - catch (Exception e) - { - LOGGER.info("migration stopped due to error for: [" + levelId + "]-[" + this.saveDir + "], error: [" + e.getMessage() + "].", e); - this.showMigrationEndMessage(false); - this.migrationStoppedWithError = true; - } - finally - { - if (this.migrationThreadRunning.get()) - { - LOGGER.info("migration complete for: [" + levelId + "]-[" + this.saveDir + "]."); - this.showMigrationEndMessage(true); - this.migrationCount = 0; - } - else - { - LOGGER.info("migration stopped for: [" + levelId + "]-[" + this.saveDir + "]."); - this.showMigrationEndMessage(false); - this.migrationStoppedWithError = true; - } - } - } - else - { - LOGGER.info("No migration necessary."); - } - } - finally - { - this.migrationThreadRunning.set(false); - } - } - - public long getLegacyDeletionCount() { return this.legacyDeletionCount; } - public long getTotalMigrationCount() { return this.migrationCount; } - public boolean getMigrationStoppedWithError() { return this.migrationStoppedWithError; } - - - private void showMigrationStartMessage() - { - if (this.migrationStartMessageQueued) - { - return; - } - this.migrationStartMessageQueued = true; - - String levelId = this.level.getLevelWrapper().getDhIdentifier(); - ClientApi.INSTANCE.showChatMessageNextFrame( - "Old Distant Horizons data is being migrated for ["+levelId+"]. \n" + - "While migrating LODs may load slowly \n" + - "and DH world gen will be disabled. \n" + - "You can see migration progress in the F3 menu." - ); - } - - private void showMigrationEndMessage(boolean success) - { - String levelId = this.level.getLevelWrapper().getDhIdentifier(); - - if (success) - { - ClientApi.INSTANCE.showChatMessageNextFrame("Distant Horizons data migration for ["+levelId+"] completed."); - } - else - { - ClientApi.INSTANCE.showChatMessageNextFrame( - "Distant Horizons data migration for ["+levelId+"] stopped. \n" + - "Some data may not have been migrated." - ); - } - } - - - - //=======================// - // retrieval (world gen) // - //=======================// - - /** - * Returns true if this provider can generate or retrieve - * {@link FullDataSourceV2}'s that aren't currently in the database. - */ - public boolean canRetrieveMissingDataSources() - { - // the base handler just handles basic reading/writing - // to the database and as such can't retrieve anything else. - return false; - } - - /** - * Returns false if this provider isn't accepting new requests, - * this can be due to having a full queue or some other - * limiting factor.

- * - * Note: when overriding make sure to add:
- * - * if (!super.canQueueRetrieval())
- * {
- * return false;
- * }
- * - * to the beginning of your override. - * Otherwise, parent retrieval limits will be ignored. - */ - public boolean canQueueRetrieval() - { - // Retrieval shouldn't happen while an unknown number of - // legacy data sources are present. - // If retrieval was allowed we might run into concurrency issues. - return !this.migrationThreadRunning.get(); - } - - /** - * @return null if this provider can't generate any positions and - * an empty array if all positions were generated - */ - @Nullable - public LongArrayList getPositionsToRetrieve(Long pos) { return null; } - - /** @return true if the position was queued, false if not */ - @Nullable - public CompletableFuture queuePositionForRetrieval(Long genPos) { return null; } - - /** does nothing if the given position isn't present in the queue */ - public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf) { } - - public void clearRetrievalQueue() { } - - /** Can be used to display how many total retrieval requests might be available. */ - public void setTotalRetrievalPositionCount(int newCount) { } - /** Can be used to display how many total chunk retrieval requests should be available. */ - public void setEstimatedRemainingRetrievalChunkCount(int newCount) { } - - public boolean fileExists(long pos) { return this.repo.getDataSizeInBytes(pos) > 0; } - - - - //========================// - // multiplayer networking // - //========================// - - @Nullable - public Long getTimestampForPos(long pos) - { return this.repo.getTimestampForPos(pos); } - - - - //===========// - // overrides // - //===========// - - @Override - public void debugRender(DebugRenderer renderer) - { - this.lockedPosSet - .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 74f, 0.15f, Color.PINK)); }); - - this.queuedUpdateCountsByPos - .forEach((pos, updateCountRef) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f + (updateCountRef.get() * 16f), 0.20f, Color.WHITE)); }); - this.updatingPosSet - .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f, 0.20f, Color.MAGENTA)); }); - } - - @Override - public void close() - { - super.close(); - if (this.updateQueueProcessor != null) - { - this.updateQueueProcessor.shutdownNow(); - } - - this.legacyFileHandler.close(); - - this.migrationThreadRunning.set(false); - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 61c8cb535..7d21a5898 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -23,6 +23,8 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataUpdatePropagatorV2; import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.generation.DhLightingEngine; import com.seibel.distanthorizons.core.generation.IFullDataSourceRetrievalQueue; @@ -41,7 +43,6 @@ import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; -import com.seibel.distanthorizons.core.logging.DhLogger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -80,7 +81,16 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im //=============// public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure) { super(level, saveStructure); } - public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } + public GeneratedFullDataSourceProvider(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) + { + super(level, saveStructure, saveDirOverride); + + this.addDataSourceUpdateListener((@NotNull FullDataSourceV2 updatedData) -> + { + this.onWorldGenTaskComplete(WorldGenResult.CreateSuccess(updatedData.getPos()), null); + }); + + } @@ -177,7 +187,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im { boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); - LOGGER.info("Set world gen queue for level [" + this.level.getLevelWrapper().getDhIdentifier() + "]."); + LOGGER.info("Set world gen queue for level [" + this.levelId + "]."); } @Override @@ -213,7 +223,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im PriorityTaskPicker.Executor renderLoadExecutor = ThreadPoolUtil.getRenderLoadingExecutor(); if (renderLoadExecutor == null - || renderLoadExecutor.getQueueSize() >= getMaxUpdateTaskCount() / 2) + || renderLoadExecutor.getQueueSize() >= FullDataUpdatePropagatorV2.getMaxPropagateTaskCount() / 2) { // don't queue additional world gen requests if the render loader handler is overwhelmed, // otherwise LODs may not load in properly @@ -222,7 +232,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im PriorityTaskPicker.Executor fileHandlerExecutor = ThreadPoolUtil.getFileHandlerExecutor(); if (fileHandlerExecutor == null - || fileHandlerExecutor.getQueueSize() >= getMaxUpdateTaskCount() / 2) + || fileHandlerExecutor.getQueueSize() >= FullDataUpdatePropagatorV2.getMaxPropagateTaskCount() / 2) { // don't queue additional world gen requests if the file handler is overwhelmed, // otherwise LODs may not load in properly @@ -294,17 +304,6 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im return worldGenFuture; } - @Override - protected void updateDataSourceAtPos(long updatePos, @NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos) - { - super.updateDataSourceAtPos(updatePos, inputData, lockOnUpdatePos); - - //if (SharedApi.getEnvironment() != EWorldEnvironment.CLIENT_ONLY) - // LOGGER.info("updated ["+DhSectionPos.toString(updatePos)+"]"); - - this.onWorldGenTaskComplete(WorldGenResult.CreateSuccess(updatePos), null); - } - @Override public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IDataSourceUpdateListenerFunc.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IDataSourceUpdateListenerFunc.java new file mode 100644 index 000000000..a5e542539 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IDataSourceUpdateListenerFunc.java @@ -0,0 +1,7 @@ +package com.seibel.distanthorizons.core.file.fullDatafile; + +@FunctionalInterface +public interface IDataSourceUpdateListenerFunc +{ + void OnDataSourceUpdated(TDataSource updatedFullDataSource); +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java similarity index 95% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java index 1527eef46..2124a90a7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V1/FullDataSourceProviderV1.java @@ -1,4 +1,4 @@ -package com.seibel.distanthorizons.core.file.fullDatafile; +package com.seibel.distanthorizons.core.file.fullDatafile.V1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.file.structure.ISaveStructure; @@ -43,10 +43,10 @@ public class FullDataSourceProviderV1 // constructor // //=============// - public FullDataSourceProviderV1(TDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) + public FullDataSourceProviderV1(TDhLevel level, File saveDir) { this.level = level; - this.saveDir = (saveDirOverride == null) ? saveStructure.getSaveFolder(level.getLevelWrapper()) : saveDirOverride; + this.saveDir = saveDir; if (!this.saveDir.exists() && !this.saveDir.mkdirs()) { LOGGER.warn("Unable to create full data folder, file saving may fail."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/DataMigratorV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/DataMigratorV1.java new file mode 100644 index 000000000..fef122cce --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/DataMigratorV1.java @@ -0,0 +1,346 @@ +package com.seibel.distanthorizons.core.file.fullDatafile.V2; + +import com.seibel.distanthorizons.core.api.internal.ClientApi; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V1.FullDataSourceProviderV1; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLogger; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.logging.f3.F3Screen; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; + +import java.io.File; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +public class DataMigratorV1 implements IDebugRenderable, AutoCloseable +{ + private static final DhLogger LOGGER = new DhLoggerBuilder().build(); + + /** how many data sources should be pulled down for migration at once */ + private static final int MIGRATION_BATCH_COUNT = FullDataUpdatePropagatorV2.NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD; + /** + * 5 minutes
+ * This should be much longer than any update should take. This is just + * to make sure the thread doesn't get stuck. + */ + private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000; + + + + private final FullDataUpdaterV2 dataUpdater; + + + private boolean migrationStartMessageQueued = false; + + private long legacyDeletionCount = -1; + private long migrationCount = -1; + private boolean migrationStoppedWithError = false; + + /** + * Interrupting the migration thread pool doesn't work well and may corrupt the database + * vs gracefully shutting down the thread ourselves. + */ + public final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); + private final FullDataSourceProviderV1 v1DataSourceProvider; + + private final String levelId; + private final File saveDir; + + + + //=============// + // constructor // + //=============// + + public DataMigratorV1( + FullDataUpdaterV2 dataUpdater, + IDhLevel level, String levelId, File saveDir) + { + this.dataUpdater = dataUpdater; + this.saveDir = saveDir; + this.v1DataSourceProvider = new FullDataSourceProviderV1<>(level, saveDir); + + this.levelId = levelId; + + + // start migrating any legacy data sources present in the background + ThreadPoolExecutor executor = ThreadPoolUtil.getFullDataMigrationExecutor(); + if (executor != null) + { + executor.execute(this::convertLegacyDataSources); + } + else + { + // shouldn't happen, but just in case + LOGGER.error("Unable to start migration for level: ["+this.levelId+"] due to missing executor."); + } + + } + + + //=======================// + // data source migration // + //=======================// + + private void convertLegacyDataSources() + { + try + { + LOGGER.debug("Attempting to migrate data sources for: [" + this.levelId + "]-[" + this.saveDir + "]..."); + this.migrationThreadRunning.set(true); + + + + //============================// + // delete unused data sources // + //============================// + + // this could be done all at once via SQL, + // but doing it in chunks prevents locking the database for long periods of time + long unusedCount = 0; + long totalDeleteCount = this.v1DataSourceProvider.repo.getUnusedDataSourceCount(); + if (totalDeleteCount != 0) + { + // this should only be shown once per session but should be shown during + // either when the deletion or migration phases start + this.showMigrationStartMessage(); + + + LOGGER.info("deleting [" + this.levelId + "] - [" + totalDeleteCount + "] unused data sources..."); + this.legacyDeletionCount = totalDeleteCount; + + ArrayList unusedDataPosList = this.v1DataSourceProvider.repo.getUnusedDataSourcePositionStringList(50); + while (unusedDataPosList.size() != 0) + { + unusedCount += unusedDataPosList.size(); + this.legacyDeletionCount -= unusedDataPosList.size(); + + + long startTime = System.currentTimeMillis(); + + // delete batch and get next batch + this.v1DataSourceProvider.repo.deleteUnusedLegacyData(unusedDataPosList); + unusedDataPosList = this.v1DataSourceProvider.repo.getUnusedDataSourcePositionStringList(50); + + long endStart = System.currentTimeMillis(); + long deleteTime = endStart - startTime; + LOGGER.info("Deleting [" + this.levelId + "] - [" + unusedCount + "/" + totalDeleteCount + "] in [" + deleteTime + "]ms ..."); + + + // a slight delay is added to prevent accidentally locking the database when deleting a lot of rows + // (that shouldn't be the case since we're using WAL journaling, but just in case) + try + { + // use the delete time so we don't make powerful computers wait super long + // and weak computers wait no time at all + Thread.sleep(deleteTime / 2); + } + catch (InterruptedException ignore) + { + } + } + LOGGER.info("Done deleting [" + this.levelId + "] - [" + totalDeleteCount + "] unused data sources."); + + } + + + + //===========// + // migration // + //===========// + + long totalMigrationCount = this.v1DataSourceProvider.getDataSourceMigrationCount(); + this.migrationCount = totalMigrationCount; + LOGGER.debug("Found [" + totalMigrationCount + "] data sources that need migration."); + + ArrayList legacyDataSourceList = this.v1DataSourceProvider.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); + if (!legacyDataSourceList.isEmpty()) + { + this.showMigrationStartMessage(); + + try + { + // keep going until every data source has been migrated + int progressCount = 0; + while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get()) + { + NumberFormat numFormat = F3Screen.NUMBER_FORMAT; + LOGGER.info("Migrating [" + this.levelId + "] - [" + numFormat.format(progressCount) + "/" + numFormat.format(totalMigrationCount) + "]..."); + + ArrayList> updateFutureList = new ArrayList<>(); + for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) + { + FullDataSourceV1 legacyDataSource = legacyDataSourceList.get(i); + + try + { + // convert the legacy data source to the new format, + // this is a relatively cheap operation + FullDataSourceV2 newDataSource = FullDataSourceV2.createFromLegacyDataSourceV1(legacyDataSource); + newDataSource.applyToParent = true; + + // the actual update process can be moderately expensive due to having to update + // the render data along with the full data, so running it async on the update threads gains us a good bit of speed + CompletableFuture future = this.dataUpdater.updateDataSourceAsync(newDataSource); + updateFutureList.add(future); + future.thenRun(() -> + { + // after the update finishes the legacy data source can be safely deleted + this.v1DataSourceProvider.repo.deleteWithKey(legacyDataSource.getPos()); + newDataSource.close(); + }); + } + catch (Exception e) + { + long migrationPos = legacyDataSource.getPos(); + LOGGER.warn("Unexpected issue migrating data source at pos [" + DhSectionPos.toString(migrationPos) + "]. Error: " + e.getMessage(), e); + this.v1DataSourceProvider.markMigrationFailed(migrationPos); + } + } + + + try + { + // wait for each thread to finish updating + CompletableFuture combinedFutures = CompletableFuture.allOf(updateFutureList.toArray(new CompletableFuture[0])); + combinedFutures.get(MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS); + } + catch (InterruptedException | TimeoutException e) + { + LOGGER.warn("Migration update timed out after [" + MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS + "] milliseconds. Migration will re-try the same positions again in a moment.", e); + } + catch (ExecutionException e) + { + LOGGER.warn("Migration update failed. Migration will re-try the same positions again. Error:" + e.getMessage(), e); + } + + legacyDataSourceList = this.v1DataSourceProvider.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); + + progressCount += legacyDataSourceList.size(); + this.migrationCount -= legacyDataSourceList.size(); + } + } + catch (Exception e) + { + LOGGER.info("migration stopped due to error for: [" + this.levelId + "]-[" + this.saveDir + "], error: [" + e.getMessage() + "].", e); + this.showMigrationEndMessage(false); + this.migrationStoppedWithError = true; + } + finally + { + if (this.migrationThreadRunning.get()) + { + LOGGER.info("migration complete for: [" + this.levelId + "]-[" + this.saveDir + "]."); + this.showMigrationEndMessage(true); + this.migrationCount = 0; + } + else + { + LOGGER.info("migration stopped for: [" + this.levelId + "]-[" + this.saveDir + "]."); + this.showMigrationEndMessage(false); + this.migrationStoppedWithError = true; + } + } + } + else + { + LOGGER.info("No migration necessary."); + } + } + finally + { + this.migrationThreadRunning.set(false); + } + } + + + private void showMigrationStartMessage() + { + if (this.migrationStartMessageQueued) + { + return; + } + this.migrationStartMessageQueued = true; + + ClientApi.INSTANCE.showChatMessageNextFrame( + "Old Distant Horizons data is being migrated for ["+this.levelId+"]. \n" + + "While migrating LODs may load slowly \n" + + "and DH world gen will be disabled. \n" + + "You can see migration progress in the F3 menu." + ); + } + + private void showMigrationEndMessage(boolean success) + { + if (success) + { + ClientApi.INSTANCE.showChatMessageNextFrame("Distant Horizons data migration for ["+this.levelId+"] completed."); + } + else + { + ClientApi.INSTANCE.showChatMessageNextFrame( + "Distant Horizons data migration for ["+this.levelId+"] stopped. \n" + + "Some data may not have been migrated." + ); + } + } + + + + //===========// + // debugging // + //===========// + + public void addDebugMenuStringsToList(List messageList) + { + // migration + boolean migrationErrored = this.migrationStoppedWithError; + if (!migrationErrored) + { + long legacyDeletionCount = this.legacyDeletionCount; + if (legacyDeletionCount > 0) + { + messageList.add(" Migrating - Deleting #: " + F3Screen.NUMBER_FORMAT.format(legacyDeletionCount)); + } + + long migrationCount = this.migrationCount; + if (migrationCount > 0) + { + messageList.add(" Migrating - Conversion #: " + F3Screen.NUMBER_FORMAT.format(migrationCount)); + } + } + else + { + messageList.add(" Migration Failed"); + } + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + // nothing currently needed + } + + @Override + public void close() + { + //LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java new file mode 100644 index 000000000..56e697aeb --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -0,0 +1,396 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.file.fullDatafile.V2; + +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc; +import com.seibel.distanthorizons.core.file.structure.ISaveStructure; +import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLogger; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; +import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Handles reading/writing {@link FullDataSourceV2} + * to and from the database. + */ +public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable +{ + private static final DhLogger LOGGER = new DhLoggerBuilder().build(); + + private static final Set CORRUPT_DATA_ERRORS_LOGGED = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + /** + * The highest numerical detail level possible. + * Used when determining which positions to update. + * + * @see FullDataSourceProviderV2#MIN_SECTION_DETAIL_LEVEL + */ + public static final byte TOP_SECTION_DETAIL_LEVEL + = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + + LodUtil.REGION_DETAIL_LEVEL; // TODO how big do we need to go? + /** + * The lowest numerical detail level possible. + * + * @see FullDataSourceProviderV2#TOP_SECTION_DETAIL_LEVEL + */ + public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + + + + protected final ReentrantLock closeLock = new ReentrantLock(); + protected volatile boolean isShutdown = false; + + protected final File saveDir; + + public final FullDataSourceV2Repo repo; + + protected final IDhLevel level; + protected final String levelId; + + + private final FullDataUpdaterV2 dataUpdater; + private final FullDataUpdatePropagatorV2 updatePropagator; + private final DataMigratorV1 dataMigratorV1; + + + + //=============// + // constructor // + //=============// + + public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure) { this(level, saveStructure, null); } + public FullDataSourceProviderV2(IDhLevel level, ISaveStructure saveStructure, @Nullable File saveDirOverride) + { + this.saveDir = (saveDirOverride == null) ? saveStructure.getSaveFolder(level.getLevelWrapper()) : saveDirOverride; + this.repo = this.createRepo(); + this.level = level; + + this.levelId = this.level.getLevelWrapper().getDhIdentifier(); + + this.dataUpdater = new FullDataUpdaterV2(this, this.levelId); + this.updatePropagator = new FullDataUpdatePropagatorV2(this, this.dataUpdater, this.levelId); + this.dataMigratorV1 = new DataMigratorV1(this.dataUpdater, this.level, this.levelId, this.saveDir); + + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus); + + } + private FullDataSourceV2Repo createRepo() + { + try + { + return new FullDataSourceV2Repo(AbstractDhRepo.DEFAULT_DATABASE_TYPE, new File(this.saveDir.getPath() + File.separator + ISaveStructure.DATABASE_NAME)); + } + catch (SQLException e) + { + // should only happen if there is an issue with the database (it's locked or the folder path is missing) + // or the database update failed + throw new RuntimeException(e); + } + } + + + + //====================// + // Abstract overrides // + //====================// + + public FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource) + { + try + { + // when creating new data use the compressor currently selected in the config + EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get(); + return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum); + } + catch (IOException e) + { + LOGGER.warn("Unable to create DTO, error: "+e.getMessage(), e); + return null; + } + } + + protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException + { return dto.createDataSource(this.level.getLevelWrapper()); } + + protected FullDataSourceV2 makeEmptyDataSource(long pos) + { return FullDataSourceV2.createEmpty(pos); } + + + + //=================// + // event listeners // + //=================// + + public void addDataSourceUpdateListener(IDataSourceUpdateListenerFunc listener) + { + synchronized (this.dataUpdater) + { + this.dataUpdater.dateSourceUpdateListeners.add(listener); + } + } + public void removeDataSourceUpdateListener(IDataSourceUpdateListenerFunc listener) + { + synchronized (this.dataUpdater) + { + this.dataUpdater.dateSourceUpdateListeners.add(listener); + } + } + + + + //=========================// + // basic DataSource getter // + //=========================// + + /** + * Returns the {@link FullDataSourceV2} for the given section position.
+ * The returned data source may be null if repo is in the process of shutting down.

+ * + * This call is concurrent. I.e. it supports being called by multiple threads at the same time. + */ + public CompletableFuture getAsync(long pos) + { + AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + + try + { + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + catch (RejectedExecutionException ignore) + { + // the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back + return CompletableFuture.completedFuture(null); + } + } + /** + * Should only be used in internal file handler methods where we are already running on a file handler thread. + * Can return null if the repo is in the process of being shut down + * @see FullDataSourceProviderV2#getAsync(long) + */ + @Nullable + public FullDataSourceV2 get(long pos) + { + try(FullDataSourceV2DTO dto = this.repo.getByKey(pos)) + { + if (dto == null) + { + return this.makeEmptyDataSource(pos); + } + + try + { + // load from database + return this.createDataSourceFromDto(dto); + } + catch (DataCorruptedException e) + { + this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); + this.repo.deleteWithKey(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); + } + + // an error occurred + return null; + } + + protected void tryLogCorruptedDataError(String whereClause, Exception e) + { + // there's a rare issue where the exception doesn't + // have a message, which can cause problems + String message = (e.getMessage() == null) ? e.getMessage() : "No Error message for exception ["+e.getClass().getSimpleName()+"]"; + + // Only log each message type once. + // This is done to prevent logging "No compression mode with the value [2]" 10,000 times + // if the user is migrating from a nightly build and used ZStd. + if (CORRUPT_DATA_ERRORS_LOGGED.add(message)) + { + LOGGER.warn("Corrupted data found at [" + whereClause + "]. Data at will be deleted so it can be re-generated to prevent issues. Future errors with this same message won't be logged. Error: [" + message + "].", e); + } + } + + + + //=======================// + // retrieval (world gen) // + //=======================// + + /** + * Returns true if this provider can generate or retrieve + * {@link FullDataSourceV2}'s that aren't currently in the database. + */ + public boolean canRetrieveMissingDataSources() + { + // the base handler just handles basic reading/writing + // to the database and as such can't retrieve anything else. + return false; + } + + /** + * Returns false if this provider isn't accepting new requests, + * this can be due to having a full queue or some other + * limiting factor.

+ * + * Note: when overriding make sure to add:
+ * + * if (!super.canQueueRetrieval())
+ * {
+ * return false;
+ * }
+ * + * to the beginning of your override. + * Otherwise, parent retrieval limits will be ignored. + */ + public boolean canQueueRetrieval() + { + // Retrieval shouldn't happen while an unknown number of + // legacy data sources are present. + // If retrieval was allowed we might run into concurrency issues. + return !this.dataMigratorV1.migrationThreadRunning.get(); + } + + /** + * @return null if this provider can't generate any positions and + * an empty array if all positions were generated + */ + @Nullable + public LongArrayList getPositionsToRetrieve(Long pos) { return null; } + + /** @return true if the position was queued, false if not */ + @Nullable + public CompletableFuture queuePositionForRetrieval(Long genPos) { return null; } + + /** does nothing if the given position isn't present in the queue */ + public void removeRetrievalRequestIf(DhSectionPos.ICancelablePrimitiveLongConsumer removeIf) { } + + public void clearRetrievalQueue() { } + + /** Can be used to display how many total retrieval requests might be available. */ + public void setTotalRetrievalPositionCount(int newCount) { } + /** Can be used to display how many total chunk retrieval requests should be available. */ + public void setEstimatedRemainingRetrievalChunkCount(int newCount) { } + + + + // + // TODO + // + + public CompletableFuture updateDataSourceAsync(@NotNull FullDataSourceV2 inputData) + { + return this.dataUpdater.updateDataSourceAsync(inputData); + } + + + + //========================// + // multiplayer networking // + //========================// + + @Nullable + public Long getTimestampForPos(long pos) + { return this.repo.getTimestampForPos(pos); } + + + + //===========// + // debugging // + //===========// + + public void addDebugMenuStringsToList(List messageList) + { + this.dataMigratorV1.addDebugMenuStringsToList(messageList); + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + this.dataUpdater.debugRender(renderer); + this.updatePropagator.debugRender(renderer); + this.dataMigratorV1.debugRender(renderer); + } + + @Override + public void close() + { + try + { + LOGGER.debug("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); + + this.closeLock.lock(); + this.isShutdown = true; + + this.dataUpdater.close(); + this.updatePropagator.close(); + this.dataMigratorV1.close(); + + // wait a moment so any queued saves can finish queuing, + // otherwise we might not see everything that needs saving and attempt to use a closed repo + Thread.sleep(200); + + this.repo.close(); + } + catch (InterruptedException ignore) { } + finally + { + this.closeLock.unlock(); + } + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java new file mode 100644 index 000000000..8f47ac19f --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java @@ -0,0 +1,408 @@ +package com.seibel.distanthorizons.core.file.fullDatafile.V2; + +import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; +import com.seibel.distanthorizons.core.logging.DhLogger; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; +import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import org.jetbrains.annotations.Nullable; + +import java.awt.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.locks.ReentrantLock; + +public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseable +{ + private static final DhLogger LOGGER = new DhLoggerBuilder().build(); + + private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); + + /** indicates how long the update queue thread should wait between queuing ticks */ + protected static final int PROPAGATE_QUEUE_THREAD_DELAY_IN_MS = 250; + + public static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 5; + + /** how many parent update tasks can be in the queue at once */ + public static int getMaxPropagateTaskCount() + { return NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Common.MultiThreading.numberOfThreads.get(); } + + + private final FullDataSourceProviderV2 provider; + private final FullDataUpdaterV2 dataUpdater; + + + /** + * Tracks which positions are currently being updated + * to prevent duplicate concurrent updates. + */ + private final Set updatingPosSet = ConcurrentHashMap.newKeySet(); + + // TODO only run thread if modifications happened recently + /** + * Will be null on the dedicated server since updates don't need to be propagated, + * only the highest detail level is needed. + */ + @Nullable + public final ThreadPoolExecutor updateQueueProcessor; + + private final String levelId; + + + + //=============// + // constructor // + //=============// + + public FullDataUpdatePropagatorV2(FullDataSourceProviderV2 provider, FullDataUpdaterV2 dataUpdater, String levelId) + { + this.provider = provider; + this.dataUpdater = dataUpdater; + this.levelId = levelId; + + // update propagation doesn't need to be run on the server since only the highest detail level is needed + this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Update Propagate Queue [" + this.levelId + "]"); + this.updateQueueProcessor.execute(this::runUpdateQueue); + } + + + //================// + // parent updates // + //================// + + private void runUpdateQueue() + { + while (!Thread.interrupted()) + { + try + { + Thread.sleep(PROPAGATE_QUEUE_THREAD_DELAY_IN_MS); + + PriorityTaskPicker.Executor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); + if (executor == null || executor.isTerminated()) + { + continue; + } + + // TODO it might be worth skipping this logic if no parent updates happened + + // update positions closest to the player (if not on a server) + // to make world gen appear faster + DhBlockPos targetBlockPos = DhBlockPos.ZERO; + if (MC_CLIENT != null + && MC_CLIENT.playerExists()) + { + targetBlockPos = MC_CLIENT.getPlayerBlockPos(); + } + + this.runParentUpdates(executor, targetBlockPos); + + if (Config.Common.LodBuilding.Experimental.upsampleLowerDetailLodsToFillHoles.get()) + { + this.runChildUpdates(executor, targetBlockPos); + } + + } + catch (InterruptedException ignored) + { + Thread.currentThread().interrupt(); + } + catch (Exception e) + { + LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e); + } + } + + LOGGER.info("Update thread ["+Thread.currentThread().getName()+"] terminated."); + } + /** will always apply updates */ + private void runParentUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos) + { + int maxUpdateTaskCount = getMaxPropagateTaskCount(); + + // queue parent updates + if (executor.getQueueSize() < maxUpdateTaskCount + && this.updatingPosSet.size() < maxUpdateTaskCount) + { + // get the positions that need to be applied to their parents + LongArrayList parentUpdatePosList = this.provider.repo.getPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount); + + // combine updates together based on their parent + HashMap> updatePosByParentPos = new HashMap<>(); + for (Long pos : parentUpdatePosList) + { + updatePosByParentPos.compute(DhSectionPos.getParentPos(pos), (parentPos, updatePosSet) -> + { + if (updatePosSet == null) + { + updatePosSet = new HashSet<>(); + } + updatePosSet.add(pos); + return updatePosSet; + }); + } + + // queue the updates + for (Long parentUpdatePos : updatePosByParentPos.keySet()) + { + // stop if there are already a bunch of updates queued + if (this.updatingPosSet.size() > maxUpdateTaskCount + || executor.getQueueSize() > maxUpdateTaskCount + || !this.updatingPosSet.add(parentUpdatePos)) + { + break; + } + + try + { + executor.execute(() -> + { + ReentrantLock parentWriteLock = this.dataUpdater.updateLockProvider.getLock(parentUpdatePos); + boolean parentLocked = false; + try + { + //LOGGER.info("updating parent: "+parentUpdatePos); + + // Locking the parent before the children should prevent deadlocks. + // TryLock is used instead of lock so this thread can handle a different update. + if (parentWriteLock.tryLock()) + { + parentLocked = true; + this.dataUpdater.lockedPosSet.add(parentUpdatePos); + + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) + { + // will return null if the file handler is shutting down + if (parentDataSource != null) + { + // apply each child pos to the parent + for (Long childPos : updatePosByParentPos.get(parentUpdatePos)) + { + ReentrantLock childReadLock = this.dataUpdater.updateLockProvider.getLock(childPos); + try + { + childReadLock.lock(); + this.dataUpdater.lockedPosSet.add(childPos); + + try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) + { + // can return null when the file handler is being shut down + if (childDataSource != null) + { + parentDataSource.updateFromChunk(childDataSource); + } + } + } + catch (Exception e) + { + LOGGER.error("Unexpected in parent update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e); + } + finally + { + childReadLock.unlock(); + this.dataUpdater.lockedPosSet.remove(childPos); + } + } + + + if (DhSectionPos.getDetailLevel(parentUpdatePos) < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL) + { + parentDataSource.applyToParent = true; + } + + this.dataUpdater.updateDataSource(parentDataSource, false); + for (Long childPos : updatePosByParentPos.get(parentUpdatePos)) + { + this.provider.repo.setApplyToParent(childPos, false); + } + } + } + } + } + finally + { + if (parentLocked) + { + parentWriteLock.unlock(); + this.dataUpdater.lockedPosSet.remove(parentUpdatePos); + } + + this.updatingPosSet.remove(parentUpdatePos); + } + }); + } + catch (RejectedExecutionException ignore) + { /* the executor was shut down, it should be back up shortly and able to accept new jobs */ } + catch (Exception e) + { + this.updatingPosSet.remove(parentUpdatePos); + throw e; + } + } + } + } + /** stops if it finds any LOD data */ + private void runChildUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos) + { + int maxUpdateTaskCount = getMaxPropagateTaskCount(); + + // queue child updates + if (executor.getQueueSize() < maxUpdateTaskCount + && this.updatingPosSet.size() < maxUpdateTaskCount) + { + // get the positions that need to be applied to their children + LongArrayList childUpdatePosList = this.provider.repo.getChildPositionsToUpdate(targetBlockPos.getX(), targetBlockPos.getZ(), maxUpdateTaskCount); + + // queue the updates + for (long parentUpdatePos : childUpdatePosList) + { + // stop if there are already a bunch of updates queued + if (this.updatingPosSet.size() > maxUpdateTaskCount + || executor.getQueueSize() > maxUpdateTaskCount) + { + break; + } + + // skip already updating positions + if (!this.updatingPosSet.add(parentUpdatePos)) + { + continue; + } + + try + { + executor.execute(() -> + { + ReentrantLock parentReadLock = this.dataUpdater.updateLockProvider.getLock(parentUpdatePos); + boolean parentLocked = false; + try + { + //LOGGER.info("updating parent: "+parentUpdatePos); + + // Locking the parent before the children should prevent deadlocks. + // TryLock is used instead of lock so this thread can handle a different update. + if (parentReadLock.tryLock()) + { + parentLocked = true; + this.dataUpdater.lockedPosSet.add(parentUpdatePos); + + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) + { + // will return null if the file handler is shutting down + if (parentDataSource != null) + { + // apply parent to each child + for (int i = 0; i < 4; i++) + { + long childPos = DhSectionPos.getChildByIndex(parentUpdatePos, i); + + ReentrantLock childWriteLock = this.dataUpdater.updateLockProvider.getLock(childPos); + try + { + childWriteLock.lock(); + this.dataUpdater.lockedPosSet.add(childPos); + + try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) + { + // will return null if the file handler is shutting down + if (childDataSource != null) + { + childDataSource.updateFromChunk(parentDataSource); + + // don't propagate child updates past the bottom of the tree + if (DhSectionPos.getDetailLevel(childPos) != DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL) + { + childDataSource.applyToChildren = true; + } + + this.dataUpdater.updateDataSource(childDataSource, false); + } + } + } + catch (Exception e) + { + LOGGER.error("Unexpected in child update propagation for parent pos: ["+DhSectionPos.toString(parentUpdatePos)+"], child pos: [" + DhSectionPos.toString(parentUpdatePos) + "], Error: [" + e.getMessage() + "].", e); + } + finally + { + childWriteLock.unlock(); + this.dataUpdater.lockedPosSet.remove(childPos); + } + } + + this.provider.repo.setApplyToChild(parentUpdatePos, false); + } + } + } + } + finally + { + if (parentLocked) + { + parentReadLock.unlock(); + this.dataUpdater.lockedPosSet.remove(parentUpdatePos); + } + + this.updatingPosSet.remove(parentUpdatePos); + } + }); + } + catch (RejectedExecutionException ignore) + { /* the executor was shut down, it should be back up shortly and able to accept new jobs */ } + catch (Exception e) + { + this.updatingPosSet.remove(parentUpdatePos); + throw e; + } + } + } + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + this.updatingPosSet + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f, 0.20f, Color.MAGENTA)); }); + } + + @Override + public void close() + { + try + { + //LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); + + if (this.updateQueueProcessor != null) + { + this.updateQueueProcessor.shutdownNow(); + } + + // wait a moment so any queued saves can finish queuing, + // otherwise we might not see everything that needs saving and attempt to use a closed repo + Thread.sleep(200); + } + catch (InterruptedException ignore) { } + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java new file mode 100644 index 000000000..c7c6352e9 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java @@ -0,0 +1,216 @@ +package com.seibel.distanthorizons.core.file.fullDatafile.V2; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc; +import com.seibel.distanthorizons.core.logging.DhLogger; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; +import org.jetbrains.annotations.NotNull; + +import java.awt.*; +import java.util.ArrayList; +import java.util.Set; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable +{ + private static final DhLogger LOGGER = new DhLoggerBuilder().build(); + + + private final FullDataSourceProviderV2 provider; + + + protected final PositionalLockProvider updateLockProvider = new PositionalLockProvider(); + /** + * generally just used for debugging, + * keeps track of which positions are currently locked. + */ + public final Set lockedPosSet = ConcurrentHashMap.newKeySet(); + private final ConcurrentHashMap queuedUpdateCountsByPos = new ConcurrentHashMap<>(); + + public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); + + private final String levelId; + + + + //=============// + // constructor // + //=============// + + public FullDataUpdaterV2(FullDataSourceProviderV2 provider, String levelId) + { + this.provider = provider; + this.levelId = levelId; + + } + + + + //===============// + // data updating // + //===============// + + /** + * Can be used if you don't want to lock the current thread + * Otherwise the sync version {@link FullDataUpdaterV2#updateDataSource(FullDataSourceV2, boolean)} may be a better choice. + */ + public CompletableFuture updateDataSourceAsync(@NotNull FullDataSourceV2 inputDataSource) + { + AbstractExecutorService executor = ThreadPoolUtil.getChunkToLodBuilderExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + + try + { + this.markUpdateStart(inputDataSource.getPos()); + return CompletableFuture.runAsync(() -> + { + try + { + this.updateDataSource(inputDataSource, true); + } + catch (Exception e) + { + LOGGER.error("Unexpected error in async data source update at pos: ["+ DhSectionPos.toString(inputDataSource.getPos())+"], error: ["+e.getMessage()+"].", e); + } + finally + { + this.markUpdateEnd(inputDataSource.getPos()); + } + }, executor); + } + catch (RejectedExecutionException ignore) + { + // can happen if the executor was shutdown while this task was queued + this.markUpdateEnd(inputDataSource.getPos()); + return CompletableFuture.completedFuture(null); + } + } + + /** After this method returns the inputData will be written to file. */ + public void updateDataSource(@NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos) + { + long updatePos = inputData.getPos(); + + boolean methodLocked = false; + // a lock is necessary to prevent two threads from writing to the same position at once, + // if that happens only the second update will apply and the LOD will end up with hole(s) + ReentrantLock updateLock = this.updateLockProvider.getLock(updatePos); + + try + { + if (lockOnUpdatePos) + { + methodLocked = true; + updateLock.lock(); + this.lockedPosSet.add(updatePos); + } + + + // get or create the data source + try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos)) + { + if (recipientDataSource != null) + { + boolean dataModified = recipientDataSource.updateFromChunk(inputData); + if (dataModified) + { + // save the updated data to the database + try (FullDataSourceV2DTO dto = this.provider.createDtoFromDataSource(recipientDataSource)) + { + this.provider.repo.save(dto); + } + + + for (IDataSourceUpdateListenerFunc listener : this.dateSourceUpdateListeners) + { + if (listener != null) + { + listener.OnDataSourceUpdated(recipientDataSource); + } + } + } + } + } + } + catch (Exception e) + { + LOGGER.error("Error updating pos ["+DhSectionPos.toString(updatePos)+"], error: "+e.getMessage(), e); + } + finally + { + if (methodLocked) + { + updateLock.unlock(); + this.lockedPosSet.remove(updatePos); + } + } + } + + + + //==================// + // debugger methods // + //==================// + + /** used for debugging to track which positions are queued for updating */ + private void markUpdateStart(long dataSourcePos) + { + this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) -> + { + if (atomicCount == null) + { + atomicCount = new AtomicInteger(0); + } + atomicCount.incrementAndGet(); + return atomicCount; + }); + } + /** used for debugging to track which positions are queued for updating */ + private void markUpdateEnd(long dataSourcePos) + { + this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) -> + { + if (atomicCount != null && atomicCount.decrementAndGet() <= 0) + { + atomicCount = null; + } + return atomicCount; + }); + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + this.lockedPosSet + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 74f, 0.15f, Color.PINK)); }); + + this.queuedUpdateCountsByPos + .forEach((pos, updateCountRef) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f + (updateCountRef.get() * 16f), 0.20f, Color.WHITE)); }); + } + + @Override + public void close() + { + //LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhServerLevel.java index 511bcf230..ac7cb485f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhServerLevel.java @@ -2,10 +2,9 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.multiplayer.server.FullDataSourceRequestHandler; import com.seibel.distanthorizons.core.multiplayer.server.ServerPlayerState; import com.seibel.distanthorizons.core.multiplayer.server.ServerPlayerStateManager; @@ -297,27 +296,7 @@ public abstract class AbstractDhServerLevel extends AbstractDhLevel implements I @Override public void addDebugMenuStringsToList(List messageList) { - // migration - boolean migrationErrored = this.serverside.fullDataFileHandler.getMigrationStoppedWithError(); - if (!migrationErrored) - { - long legacyDeletionCount = this.serverside.fullDataFileHandler.getLegacyDeletionCount(); - if (legacyDeletionCount > 0) - { - messageList.add(" Migrating - Deleting #: " + F3Screen.NUMBER_FORMAT.format(legacyDeletionCount)); - } - long migrationCount = this.serverside.fullDataFileHandler.getTotalMigrationCount(); - if (migrationCount > 0) - { - messageList.add(" Migrating - Conversion #: " + F3Screen.NUMBER_FORMAT.format(migrationCount)); - } - } - else - { - messageList.add(" Migration Failed"); - } - - // world gen + this.serverside.fullDataFileHandler.addDebugMenuStringsToList(messageList); this.serverside.worldGenModule.addDebugMenuStringsToList(messageList); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index b73fd8038..ba8099a52 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -22,8 +22,8 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos2D; import com.seibel.distanthorizons.core.render.LodQuadTree; @@ -39,7 +39,7 @@ import java.io.Closeable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; -public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.IDataSourceUpdateListenerFunc +public class ClientLevelModule implements Closeable, IDataSourceUpdateListenerFunc { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); @@ -69,7 +69,7 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I this.clientLevel = clientLevel; this.fullDataSourceProvider = this.clientLevel.getFullDataProvider(); - this.fullDataSourceProvider.dateSourceUpdateListeners.add(this); + this.fullDataSourceProvider.addDataSourceUpdateListener(this); } @@ -161,7 +161,8 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I // data handling // //===============// - public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientLevel.getFullDataProvider().updateDataSourceAsync(data); } + public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) + { return this.clientLevel.getFullDataProvider().updateDataSourceAsync(data); } @Override public void OnDataSourceUpdated(FullDataSourceV2 updatedFullDataSource) { @@ -195,7 +196,7 @@ public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.I } } - this.fullDataSourceProvider.dateSourceUpdateListeners.remove(this); + this.fullDataSourceProvider.removeDataSourceUpdateListener(this); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 795c0e5da..6823db53d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -23,7 +23,7 @@ import com.google.common.cache.CacheBuilder; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataSourceProvider; import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.generation.RemoteWorldRetrievalQueue; @@ -68,7 +68,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public final ClientLevelModule clientside; public final IClientLevelWrapper levelWrapper; public final ISaveStructure saveStructure; - public final RemoteFullDataSourceProvider dataFileHandler; + public final RemoteFullDataSourceProvider remoteDataSourceProvider; @CheckForNull private final ClientNetworkState networkState; @@ -130,12 +130,12 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel this.syncOnLoadRequestQueue = null; } - this.dataFileHandler = new RemoteFullDataSourceProvider(this, saveStructure, fullDataSaveDirOverride, this.syncOnLoadRequestQueue); - this.worldGenModule = new WorldGenModule(this, this.dataFileHandler, () -> new WorldGenState(this, networkState)); + this.remoteDataSourceProvider = new RemoteFullDataSourceProvider(this, saveStructure, fullDataSaveDirOverride, this.syncOnLoadRequestQueue); + this.worldGenModule = new WorldGenModule(this, this.remoteDataSourceProvider, () -> new WorldGenState(this, networkState)); this.clientside = new ClientLevelModule(this); - this.createAndSetSupportingRepos(this.dataFileHandler.repo.databaseFile); + this.createAndSetSupportingRepos(this.remoteDataSourceProvider.repo.databaseFile); this.runRepoReliantSetup(); this.clientside.startRenderer(); @@ -283,7 +283,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientside.updateDataSourcesAsync(data); } @Override - public FullDataSourceProviderV2 getFullDataProvider() { return this.dataFileHandler; } + public FullDataSourceProviderV2 getFullDataProvider() { return this.remoteDataSourceProvider; } @Override public ISaveStructure getSaveStructure() { return this.saveStructure; } @@ -321,24 +321,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel messageList.add("["+dimName+"] rendering: "+(rendering ? "yes" : "no")); - boolean migrationErrored = this.dataFileHandler.getMigrationStoppedWithError(); - if (!migrationErrored) - { - long legacyDeletionCount = this.dataFileHandler.getLegacyDeletionCount(); - if (legacyDeletionCount > 0) - { - messageList.add(" Migrating - Deleting #: " + legacyDeletionCount); - } - long migrationCount = this.dataFileHandler.getTotalMigrationCount(); - if (migrationCount > 0) - { - messageList.add(" Migrating - Conversion #: " + migrationCount); - } - } - else - { - messageList.add(" Migration Failed"); - } + this.remoteDataSourceProvider.addDebugMenuStringsToList(messageList); // world gen @@ -378,7 +361,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel this.levelWrapper.setDhLevel(null); this.clientside.close(); super.close(); - this.dataFileHandler.close(); + this.remoteDataSourceProvider.close(); LOGGER.info("Closed [" + DhClientLevel.class.getSimpleName() + "] for [" + this.levelWrapper + "]"); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java index 656cb1044..7178111cc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.core.api.internal.ClientApi; import com.seibel.distanthorizons.core.api.internal.ServerApi; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider; import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.pos.DhChunkPos; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 154a0cdb7..4b87eeeba 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -26,7 +26,7 @@ import com.seibel.distanthorizons.core.dataObjects.render.CachedColumnRenderSour import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer; import com.seibel.distanthorizons.core.enums.EDhDirection; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index aecf09bcc..49c52f870 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -31,7 +31,7 @@ import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodQuad import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; From 5fd8ed840f2b8b1348f11aa54da4286edf879ae9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 6 Nov 2025 07:35:23 -0600 Subject: [PATCH 07/24] Add adjacent data to FullDataDTO for faster loading --- .../methods/data/DhApiTerrainDataRepo.java | 2 +- .../render/CachedColumnRenderSource.java | 96 ------- .../FullDataToRenderDataTransformer.java | 6 +- .../GeneratedFullDataSourceProvider.java | 2 +- .../RemoteFullDataSourceProvider.java | 8 +- .../V2/FullDataSourceProviderV2.java | 120 ++++++--- .../V2/FullDataUpdatePropagatorV2.java | 8 +- .../fullDatafile/V2/FullDataUpdaterV2.java | 28 +- .../core/generation/PregenManager.java | 4 +- .../distanthorizons/core/jar/JarMain.java | 2 +- .../core/level/DhClientLevel.java | 2 +- .../AbstractFullDataNetworkRequestQueue.java | 2 +- .../server/FullDataSourceRequestHandler.java | 4 +- .../core/render/LodQuadTree.java | 86 +----- .../core/render/LodRenderSection.java | 74 ++---- .../core/sql/dto/FullDataSourceV2DTO.java | 165 +++++++++++- .../sql/dto/util/FullDataMinMaxPosUtil.java | 108 ++++++++ .../core/sql/repo/FullDataSourceV2Repo.java | 249 +++++++++++++++++- ...0090-sqlite-addAdjacentFullDataColumns.sql | 13 + .../main/resources/sqlScripts/scriptList.txt | 1 + .../java/tests/DhFullDataSourceRepoTests.java | 123 +++++++-- .../java/tests/FullDataMinMaxPosTest.java | 54 ++++ 22 files changed, 834 insertions(+), 323 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java create mode 100644 core/src/main/resources/sqlScripts/0090-sqlite-addAdjacentFullDataColumns.sql create mode 100644 core/src/test/java/tests/FullDataMinMaxPosTest.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index c27b8a58e..0dd8f2d15 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -238,7 +238,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo if (dataSource == null) { // attempt to get/generate the data source for this section - dataSource = level.getFullDataProvider().getAsync(sectionPos).get(); + dataSource = level.getFullDataProvider().getAsync(sectionPos, false).get(); if (dataSource == null) { return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + DhSectionPos.toString(sectionPos) + "]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java deleted file mode 100644 index 330186ae5..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/CachedColumnRenderSource.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.seibel.distanthorizons.core.dataObjects.render; - -import com.google.common.cache.Cache; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Wrapper for {@link ColumnRenderSource} that handles reference counting - * and cache tracking. - */ -public class CachedColumnRenderSource implements AutoCloseable -{ - /** an externally handled future that will complete once the {@link CachedColumnRenderSource#columnRenderSource} has finished loading */ - public final CompletableFuture loadFuture; - /** will be null initially, should be non-null once the corresponding load future is done */ - @Nullable - public ColumnRenderSource columnRenderSource = null; - - private final AtomicInteger referenceCount; - private final Cache cachedRenderSourceByPos; - private final ReentrantLock getterLock; - - - - //=============// - // constructor // - //=============// - - public CachedColumnRenderSource( - @NotNull CompletableFuture loadFuture, - @NotNull ReentrantLock getterLock, - @NotNull Cache cachedRenderSourceByPos) - { - this.loadFuture = loadFuture; - this.getterLock = getterLock; - this.referenceCount = new AtomicInteger(1); - this.cachedRenderSourceByPos = cachedRenderSourceByPos; - } - - - - //====================// - // reference counting // - //====================// - - public void markInUse() { this.referenceCount.getAndIncrement(); } - - - - //================// - // base overrides // - //================// - - /** - * Will be called multiple times, - * however it will only close the underlying data once - * all references have closed. - */ - @Override - public void close() throws IllegalStateException - { - try - { - // lock to prevent other threads for accessing the cache if we invalidate it - this.getterLock.lock(); - - // should only happen if something goes wrong up-stream - if (this.columnRenderSource == null) - { - return; - } - - - // only close once everyone is done with this datasource - int refCount = this.referenceCount.decrementAndGet(); - if (refCount == 0) - { - this.cachedRenderSourceByPos.invalidate(this.columnRenderSource.pos); - this.columnRenderSource.close(); - } - else if (refCount < 0) - { - throw new IllegalStateException("Render source ["+this.columnRenderSource.pos+"] reference count incorrect. Object already closed."); - } - } - finally - { - this.getterLock.unlock(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 5d43cea48..52fa961e0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -66,7 +66,8 @@ public class FullDataToRenderDataTransformer //==============================// @Nullable - public static ColumnRenderSource transformFullDataToRenderSource(@Nullable FullDataSourceV2 fullDataSource, @Nullable IClientLevelWrapper levelWrapper) + public static ColumnRenderSource transformFullDataToRenderSource( + @Nullable FullDataSourceV2 fullDataSource, @Nullable IClientLevelWrapper levelWrapper) { if (fullDataSource == null) { @@ -102,7 +103,8 @@ public class FullDataToRenderDataTransformer * @throws InterruptedException Can be caused by interrupting the thread upstream. * Generally thrown if the method is running after the client leaves the current world. */ - private static ColumnRenderSource transformCompleteFullDataToColumnData(IClientLevelWrapper levelWrapper, FullDataSourceV2 fullDataSource) throws InterruptedException + private static ColumnRenderSource transformCompleteFullDataToColumnData( + IClientLevelWrapper levelWrapper, FullDataSourceV2 fullDataSource) throws InterruptedException { final long pos = fullDataSource.getPos(); final byte dataDetail = fullDataSource.getDataDetailLevel(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 7d21a5898..47040c4a7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -489,7 +489,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im @Override public CompletableFuture shouldGenerateSplitChild(long pos) { - return GeneratedFullDataSourceProvider.this.getAsync(pos).thenApply(fullDataSource -> + return GeneratedFullDataSourceProvider.this.getAsync(pos, false).thenApply(fullDataSource -> { //noinspection TryFinallyCanBeTryWithResources try diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java index b3dbae741..618523718 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java @@ -75,18 +75,18 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide @Override @Nullable - public FullDataSourceV2 get(long pos) + public FullDataSourceV2 get(long pos, boolean includeAdjacentData) { if (this.syncOnLoadRequestQueue == null) { // we have local data, but networking is unavailable. - return super.get(pos); + return super.get(pos, includeAdjacentData); } if (!this.visitedPositions.add(pos)) { // This position has already been accessed before - return super.get(pos); + return super.get(pos, includeAdjacentData); } @@ -105,7 +105,7 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide }); } - return super.get(pos); + return super.get(pos, includeAdjacentData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index 56e697aeb..d3d83bf23 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.file.fullDatafile.V2; -import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc; import com.seibel.distanthorizons.core.file.structure.ISaveStructure; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; @@ -129,33 +129,6 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable - //====================// - // Abstract overrides // - //====================// - - public FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource) - { - try - { - // when creating new data use the compressor currently selected in the config - EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get(); - return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum); - } - catch (IOException e) - { - LOGGER.warn("Unable to create DTO, error: "+e.getMessage(), e); - return null; - } - } - - protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException - { return dto.createDataSource(this.level.getLevelWrapper()); } - - protected FullDataSourceV2 makeEmptyDataSource(long pos) - { return FullDataSourceV2.createEmpty(pos); } - - - //=================// // event listeners // //=================// @@ -177,6 +150,17 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable + //================// + // DTO converters // + //================// + + protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException, DataCorruptedException + { return dto.createDataSource(this.level.getLevelWrapper(), null); } + protected FullDataSourceV2 createAdjDataSourceFromDto(FullDataSourceV2DTO dto, EDhDirection direction) throws InterruptedException, IOException, DataCorruptedException + { return dto.createDataSource(this.level.getLevelWrapper(), direction); } + + + //=========================// // basic DataSource getter // //=========================// @@ -187,7 +171,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable * * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ - public CompletableFuture getAsync(long pos) + public CompletableFuture getAsync(long pos, boolean includeAdjacentData) { AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) @@ -198,7 +182,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable try { - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + return CompletableFuture.supplyAsync(() -> this.get(pos, includeAdjacentData), executor); } catch (RejectedExecutionException ignore) { @@ -209,16 +193,18 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable /** * Should only be used in internal file handler methods where we are already running on a file handler thread. * Can return null if the repo is in the process of being shut down - * @see FullDataSourceProviderV2#getAsync(long) + * @see FullDataSourceProviderV2#getAsync(long, boolean) */ @Nullable - public FullDataSourceV2 get(long pos) + public FullDataSourceV2 get(long pos, boolean includeAdjacentData) { - try(FullDataSourceV2DTO dto = this.repo.getByKey(pos)) + try(FullDataSourceV2DTO dto = includeAdjacentData + ? this.repo.getByKey(pos) + : this.repo.getByPosNoAdj(pos)) { if (dto == null) { - return this.makeEmptyDataSource(pos); + return FullDataSourceV2.createEmpty(pos); } try @@ -259,6 +245,72 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable + // + // TODO name? + // + + public FullDataSourceV2 getAdjForDirection(long pos, EDhDirection direction) + { + try(FullDataSourceV2DTO dto = this.repo.getAdjByPosAndDirection(pos, direction)) + { + if (dto == null) + { + return FullDataSourceV2.createEmpty(pos); + } + + try + { + // load from database + return this.createAdjDataSourceFromDto(dto, direction); + } + catch (DataCorruptedException e) + { + this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); + this.repo.deleteWithKey(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); + } + + // an error occurred + return null; + } + + public FullDataSourceV2 getCenter(long pos) + { + try(FullDataSourceV2DTO dto = this.repo.getByPosNoAdj(pos)) + { + if (dto == null) + { + return FullDataSourceV2.createEmpty(pos); + } + + try + { + // load from database + return this.createDataSourceFromDto(dto); + } + catch (DataCorruptedException e) + { + this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); + this.repo.deleteWithKey(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); + } + + // an error occurred + return null; + } + + + //=======================// // retrieval (world gen) // //=======================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java index 8f47ac19f..f1de0c42e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java @@ -183,7 +183,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab parentLocked = true; this.dataUpdater.lockedPosSet.add(parentUpdatePos); - try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos, false)) { // will return null if the file handler is shutting down if (parentDataSource != null) @@ -197,7 +197,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab childReadLock.lock(); this.dataUpdater.lockedPosSet.add(childPos); - try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) + try (FullDataSourceV2 childDataSource = this.provider.get(childPos, false)) { // can return null when the file handler is being shut down if (childDataSource != null) @@ -299,7 +299,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab parentLocked = true; this.dataUpdater.lockedPosSet.add(parentUpdatePos); - try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos, false)) { // will return null if the file handler is shutting down if (parentDataSource != null) @@ -315,7 +315,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab childWriteLock.lock(); this.dataUpdater.lockedPosSet.add(childPos); - try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) + try (FullDataSourceV2 childDataSource = this.provider.get(childPos, false)) { // will return null if the file handler is shutting down if (childDataSource != null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java index c7c6352e9..c18a651b2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java @@ -1,5 +1,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile.V2; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.file.fullDatafile.IDataSourceUpdateListenerFunc; import com.seibel.distanthorizons.core.logging.DhLogger; @@ -13,6 +15,7 @@ import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.jetbrains.annotations.NotNull; import java.awt.*; +import java.io.IOException; import java.util.ArrayList; import java.util.Set; import java.util.concurrent.*; @@ -119,7 +122,7 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable // get or create the data source - try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos)) + try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos, false)) { if (recipientDataSource != null) { @@ -127,9 +130,12 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable if (dataModified) { // save the updated data to the database - try (FullDataSourceV2DTO dto = this.provider.createDtoFromDataSource(recipientDataSource)) + try (FullDataSourceV2DTO dto = this.createDtoFromDataSource(recipientDataSource)) { - this.provider.repo.save(dto); + if (dto != null) + { + this.provider.repo.save(dto); + } } @@ -158,6 +164,22 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable } } + private FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource) + { + try + { + // when creating new data use the compressor currently selected in the config + EDhApiDataCompressionMode compressionModeEnum = Config.Common.LodBuilding.dataCompression.get(); + return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum); + } + catch (IOException e) + { + LOGGER.warn("Unable to create DTO, error: ["+e.getMessage() + "].", e); + return null; + } + } + + //==================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java index c8f368419..53cabc7ef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java @@ -140,7 +140,9 @@ public class PregenManager } this.pendingGenerations.put(nextSectionPos, System.currentTimeMillis()); - this.fullDataSourceProvider.getAsync(nextSectionPos).thenAccept(fullDataSource -> { + this.fullDataSourceProvider.getAsync(nextSectionPos, false) + .thenAccept(fullDataSource -> + { if (this.fullDataSourceProvider.isFullyGenerated(fullDataSource.columnGenerationSteps)) { this.pendingGenerations.invalidate(fullDataSource.getPos()); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java index 806b3baeb..f3ff831d3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java @@ -239,7 +239,7 @@ public class JarMain private static void exportLodDataAtPosition(FullDataSourceV2Repo repo, File exportFile, long pos) { - FullDataSourceV2DTO dto = repo.getByKey(pos); + FullDataSourceV2DTO dto = repo.getByPosNoAdj(pos); if (dto == null) { LOGGER.error("Unable to find any data at the position ["+DhSectionPos.toString(pos)+"]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 6823db53d..fef19a6da 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -182,7 +182,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel } - FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.levelWrapper); + FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.levelWrapper, null); this.updateDataSourcesAsync(fullDataSource) .whenComplete((result, e) -> fullDataSource.close()); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java index cc8294f3c..b9608e566 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/client/AbstractFullDataNetworkRequestQueue.java @@ -279,7 +279,7 @@ public abstract class AbstractFullDataNetworkRequestQueue implements IDebugRende { this.level.updateBeaconBeamsForSectionPos(dataSourceDto.pos, response.payload.beaconBeams); - FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper()); + FullDataSourceV2 fullDataSource = dataSourceDto.createDataSource(this.level.getLevelWrapper(), null); entry.dataSourceConsumer.accept(fullDataSource); } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java index 2e523ea25..0b0be6c17 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java @@ -99,7 +99,7 @@ public class FullDataSourceRequestHandler } // get the server's datasource - return this.fullDataSourceProvider().get(message.sectionPos); + return this.fullDataSourceProvider().get(message.sectionPos, false); } catch (Exception e) { @@ -262,7 +262,7 @@ public class FullDataSourceRequestHandler private void tryFulfillDataSourceRequestGroup(DataSourceRequestGroup requestGroup, long pos) { - this.fullDataSourceProvider().getAsync(pos).thenAccept(fullDataSource -> + this.fullDataSourceProvider().getAsync(pos, false).thenAccept(fullDataSource -> { if (this.fullDataSourceProvider().isFullyGenerated(fullDataSource.columnGenerationSteps)) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 4b87eeeba..fb4cdde72 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -22,7 +22,6 @@ package com.seibel.distanthorizons.core.render; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.render.CachedColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer; import com.seibel.distanthorizons.core.enums.EDhDirection; @@ -87,28 +86,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen private ArrayList altDebugRenderSections = new ArrayList<>(); private final ReentrantLock debugRenderSectionLock = new ReentrantLock(); - - /** don't let two threads load the same position at the same time */ - protected final KeyedLockContainer renderLoadLockContainer = new KeyedLockContainer<>(); - - /** - * caching is done at the QuadTree level to prevent caching LODs for different levels. - * (Although the incorrect terrain that renders is quite entertaining).

- * - * caching the loaded positions significantly improves initial loading performance - * since the same position doesn't need to be loaded 5 times. - */ - private final Cache cachedRenderSourceByPos - = CacheBuilder.newBuilder() - // availableProcessors() : each process may need to be loading a render source - // +1 : add 1 thread count buffer to reduce the chance of accidentally unloading a render source before it's used - // *5 : each render source needs it's 4 adjacent sides, so a total of 5 render sources are needed per load - .maximumSize((Runtime.getRuntime().availableProcessors() + 1) * 5L) - // No closing logic since the CachedColumnRenderSource is in charge - // of freeing the underlying ColumnRenderSource. - // That way we don't have to worry about accidentally closing an in-use object. - .build(); - /** * Used to limit how many upload tasks are queued at once. * If all the upload tasks are queued at once, they will start uploading nearest @@ -240,7 +217,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen long rootPos = rootPosIterator.nextLong(); if (this.getNode(rootPos) == null) { - this.setValue(rootPos, new LodRenderSection(rootPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer)); + this.setValue(rootPos, new LodRenderSection(rootPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef)); } QuadNode rootNode = this.getNode(rootPos); @@ -281,7 +258,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen // create the node if (quadNode == null && this.isSectionPosInBounds(sectionPos)) // the position bounds should only fail when at the edge of the user's render distance { - rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer)); + rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef)); quadNode = rootNode.getNode(sectionPos); } if (quadNode == null) @@ -294,7 +271,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen LodRenderSection renderSection = quadNode.value; if (renderSection == null) { - renderSection = new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef, this.cachedRenderSourceByPos, this.renderLoadLockContainer); + renderSection = new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider, this.uploadTaskCountRef); quadNode.setValue(sectionPos, renderSection); } @@ -636,17 +613,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen * This should be called whenever a world generation task is completed or if the connected server has new data to show. */ public void reloadPos(long pos) - { - // clear cache // - - this.clearRenderCacheForPos(pos); - for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS) - { - long adjacentPos = DhSectionPos.getAdjacentPos(pos, direction); - this.clearRenderCacheForPos(adjacentPos); - } - - + { // queue reloads // // only queue each section for reloading @@ -664,22 +631,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen this.sectionsToReload.add(adjacentPos); } } - private void clearRenderCacheForPos(long pos) - { - // locking is needed to prevent another thread - // from accessing the cache while it's being cleared - ReentrantLock lock = this.renderLoadLockContainer.getLockForPos(pos); - try - { - lock.lock(); - this.cachedRenderSourceByPos.invalidate(pos); - } - finally - { - lock.unlock(); - } - } - //=================================// @@ -830,39 +781,10 @@ public class LodQuadTree extends QuadTree implements IDebugRen LodRenderSection renderSection = quadNode.value; if (renderSection != null) { - // we need to wait for the render data to finish building before we can close the cache - CompletableFuture future = renderSection.getRenderDataBuildFuture(); - if (future != null) - { - renderDataBuildFutures.add(future); - } - renderSection.close(); quadNode.value = null; } } - - - // close the render cache after it is done being used - LOGGER.info("waiting for ["+renderDataBuildFutures.size()+"] futures before closing render cache..."); - CompletableFuture.allOf(renderDataBuildFutures.toArray(new CompletableFuture[0])) - .handle((voidObj, throwable) -> - { - // run on a separate thread so we don't lock up the main cleanup thread - // with the sleep() call - new Thread(() -> - { - // Sleep shouldn't be necessary, but James found a few cases where - // the futures incorrectly claimed they were done. - // Sleeping solved those issues. - try { Thread.sleep(5_000); } catch (InterruptedException ignore) { } - - LOGGER.debug("closing render cache"); - this.cachedRenderSourceByPos.invalidateAll(); - }).start(); - - return null; - }); } finally { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 49c52f870..9732596df 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -21,10 +21,8 @@ package com.seibel.distanthorizons.core.render; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; -import com.google.common.cache.Cache; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dataObjects.render.CachedColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodQuadBuilder; @@ -44,7 +42,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler; import com.seibel.distanthorizons.core.sql.dto.BeaconBeamDTO; import com.seibel.distanthorizons.core.sql.repo.BeaconBeamRepo; -import com.seibel.distanthorizons.core.util.KeyedLockContainer; import com.seibel.distanthorizons.core.util.PerfRecorder; import com.seibel.distanthorizons.core.util.threading.PriorityTaskPicker; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; @@ -59,7 +56,6 @@ import java.util.*; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; /** * A render section represents an area that could be rendered. @@ -79,8 +75,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable @WillNotClose private final FullDataSourceProviderV2 fullDataSourceProvider; private final LodQuadTree quadTree; - private final KeyedLockContainer renderLoadLockContainer; - private final Cache cachedRenderSourceByPos; private final AtomicInteger uploadTaskCountRef; /** @@ -147,13 +141,10 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable long pos, LodQuadTree quadTree, IDhClientLevel level, FullDataSourceProviderV2 fullDataSourceProvider, - AtomicInteger uploadTaskCountRef, - Cache cachedRenderSourceByPos, KeyedLockContainer renderLoadLockContainer) + AtomicInteger uploadTaskCountRef) { this.pos = pos; this.quadTree = quadTree; - this.cachedRenderSourceByPos = cachedRenderSourceByPos; - this.renderLoadLockContainer = renderLoadLockContainer; this.level = level; this.levelWrapper = level.getClientLevelWrapper(); this.fullDataSourceProvider = fullDataSourceProvider; @@ -245,11 +236,11 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { // get the center pos data return this.getRenderSourceForPosAsync(this.pos, null) - .thenCompose((CachedColumnRenderSource cachedRenderSource) -> + .thenCompose((ColumnRenderSource thisRenderSource) -> { try { - if (cachedRenderSource == null || cachedRenderSource.columnRenderSource == null) + if (thisRenderSource == null) { // nothing needs to be rendered // TODO how doesn't this cause infinite file handler loops? @@ -257,7 +248,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // setting the render buffer here return CompletableFuture.completedFuture(null); } - ColumnRenderSource thisRenderSource = cachedRenderSource.columnRenderSource; boolean enableTransparency = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled; @@ -268,7 +258,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // get the adjacent positions // needs to be done async to prevent threads waiting on the same positions to be processed - final CompletableFuture[] adjacentLoadFutures = new CompletableFuture[4]; + final CompletableFuture[] adjacentLoadFutures = new CompletableFuture[4]; if (Config.Client.Advanced.Graphics.Experimental.onlyLoadCenterLods.get()) { @@ -291,16 +281,16 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { getAdj.end(); - try (CachedColumnRenderSource northRenderSource = adjacentLoadFutures[0].get(); - CachedColumnRenderSource southRenderSource = adjacentLoadFutures[1].get(); - CachedColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get(); - CachedColumnRenderSource westRenderSource = adjacentLoadFutures[3].get()) + try (ColumnRenderSource northRenderSource = adjacentLoadFutures[0].get(); + ColumnRenderSource southRenderSource = adjacentLoadFutures[1].get(); + ColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get(); + ColumnRenderSource westRenderSource = adjacentLoadFutures[3].get()) { ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.CARDINAL_COMPASS.length]; - adjacentRenderSections[EDhDirection.NORTH.ordinal() - 2] = (northRenderSource != null) ? northRenderSource.columnRenderSource : null; - adjacentRenderSections[EDhDirection.SOUTH.ordinal() - 2] = (southRenderSource != null) ? southRenderSource.columnRenderSource : null; - adjacentRenderSections[EDhDirection.EAST.ordinal() - 2] = (eastRenderSource != null) ? eastRenderSource.columnRenderSource : null; - adjacentRenderSections[EDhDirection.WEST.ordinal() - 2] = (westRenderSource != null) ? westRenderSource.columnRenderSource : null; + adjacentRenderSections[EDhDirection.NORTH.ordinal() - 2] = northRenderSource; + adjacentRenderSections[EDhDirection.SOUTH.ordinal() - 2] = southRenderSource; + adjacentRenderSections[EDhDirection.EAST.ordinal() - 2] = eastRenderSource; + adjacentRenderSections[EDhDirection.WEST.ordinal() - 2] = westRenderSource; boolean[] adjIsSameDetailLevel = new boolean[EDhDirection.CARDINAL_COMPASS.length]; adjIsSameDetailLevel[EDhDirection.NORTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.NORTH); @@ -325,7 +315,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable finally { // can only be closed after the data has been processed and uploaded to the GPU - cachedRenderSource.close(); + thisRenderSource.close(); } }); } @@ -337,7 +327,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable }); } /** async is done so each thread can run without waiting on others */ - private CompletableFuture getRenderSourceForPosAsync(long pos, @Nullable EDhDirection direction) + private CompletableFuture getRenderSourceForPosAsync(long pos, @Nullable EDhDirection direction) { if (direction != null) { @@ -346,23 +336,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable final long finalPos = pos; - ReentrantLock lock = this.renderLoadLockContainer.getLockForPos(finalPos); try { - // we don't want multiple threads attempting to load the same position at the same time, - // and we don't want to access the cache while invalidating it on a different thread - lock.lock(); - - // use the cached data if possible - CachedColumnRenderSource existingCachedRenderSource = this.cachedRenderSourceByPos.getIfPresent(finalPos); - if (existingCachedRenderSource != null) - { - existingCachedRenderSource.markInUse(); - return existingCachedRenderSource.loadFuture; - } - - - PriorityTaskPicker.Executor executor = ThreadPoolUtil.getRenderLoadingExecutor(); if (executor == null || executor.isTerminated()) { @@ -372,31 +347,30 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // queue loading the render data - CompletableFuture loadFuture = new CompletableFuture<>(); - final CachedColumnRenderSource newCachedRenderSource = new CachedColumnRenderSource(loadFuture, lock, this.cachedRenderSourceByPos); + CompletableFuture loadFuture = new CompletableFuture<>(); executor.execute(() -> { PerfRecorder.Timer getFull = this.filePerfRecorder.start("getFull"); // generate new render source - try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(finalPos)) + try (FullDataSourceV2 fullDataSource = + // no direction means get the center LOD + (direction == null) + ? this.fullDataSourceProvider.getCenter(finalPos) + : this.fullDataSourceProvider.getAdjForDirection(finalPos, direction.opposite())) { getFull.end(); PerfRecorder.Timer transform = this.filePerfRecorder.start("transform"); - newCachedRenderSource.columnRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.levelWrapper); + ColumnRenderSource columnRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.levelWrapper); + loadFuture.complete(columnRenderSource); transform.end(); } catch (Exception e) { LOGGER.error("Unexpected issue creating render data for pos: ["+DhSectionPos.toString(finalPos)+"], error: ["+e.getMessage()+"].", e); } - finally - { - loadFuture.complete(newCachedRenderSource); - } }); - this.cachedRenderSourceByPos.put(pos, newCachedRenderSource); return loadFuture; } @@ -410,10 +384,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable LOGGER.error("Unexpected issue getting and creating render data for pos: ["+DhSectionPos.toString(pos)+"], error: ["+e.getMessage()+"].", e); return CompletableFuture.completedFuture(null); } - finally - { - lock.unlock(); - } } private boolean isAdjacentPosSameDetailLevel(EDhDirection direction) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index d49d40ea9..90b79d6c5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -25,10 +25,12 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.network.INetworkObject; +import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil; import com.seibel.distanthorizons.core.util.BoolUtil; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.ListUtil; @@ -61,6 +63,10 @@ public class FullDataSourceV2DTO public int dataChecksum; public ByteArrayList compressedDataByteArray; + public ByteArrayList compressedNorthAdjDataByteArray; + public ByteArrayList compressedSouthAdjDataByteArray; + public ByteArrayList compressedEastAdjDataByteArray; + public ByteArrayList compressedWestAdjDataByteArray; /** @see EDhApiWorldGenerationStep */ public ByteArrayList compressedColumnGenStepByteArray; @@ -100,6 +106,11 @@ public class FullDataSourceV2DTO writeGenerationStepsToBlob(dataSource.columnGenerationSteps, dto.compressedColumnGenStepByteArray, compressionModeEnum); writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, dto.compressedWorldCompressionModeByteArray, compressionModeEnum); writeDataMappingToBlob(dataSource.mapping, dto.compressedMappingByteArray, compressionModeEnum); + // adjacent full data + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedNorthAdjDataByteArray, EDhDirection.NORTH, compressionModeEnum); + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedSouthAdjDataByteArray, EDhDirection.SOUTH, compressionModeEnum); + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedEastAdjDataByteArray, EDhDirection.EAST, compressionModeEnum); + writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedWestAdjDataByteArray, EDhDirection.WEST, compressionModeEnum); // populate individual variables { @@ -124,7 +135,7 @@ public class FullDataSourceV2DTO private FullDataSourceV2DTO() { - super(ARRAY_LIST_POOL, 4, 0, 0); + super(ARRAY_LIST_POOL, 8, 0, 0); // Expected sizes here are 0 since we don't know how big these arrays need to be, // they depend on compression settings and world complexity. @@ -132,6 +143,11 @@ public class FullDataSourceV2DTO this.compressedColumnGenStepByteArray = this.pooledArraysCheckout.getByteArray(1, 0); this.compressedWorldCompressionModeByteArray = this.pooledArraysCheckout.getByteArray(2, 0); this.compressedMappingByteArray = this.pooledArraysCheckout.getByteArray(3, 0); + + this.compressedNorthAdjDataByteArray = this.pooledArraysCheckout.getByteArray(4, 0); + this.compressedSouthAdjDataByteArray = this.pooledArraysCheckout.getByteArray(5, 0); + this.compressedEastAdjDataByteArray = this.pooledArraysCheckout.getByteArray(6, 0); + this.compressedWestAdjDataByteArray = this.pooledArraysCheckout.getByteArray(7, 0); } @@ -140,12 +156,12 @@ public class FullDataSourceV2DTO // data source population // //========================// - public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException, DataCorruptedException + public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper, EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException { FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos); try { - this.internalPopulateDataSource(dataSource, levelWrapper, false); + this.internalPopulateDataSource(dataSource, levelWrapper, direction, false); } catch (Exception e) { @@ -156,14 +172,19 @@ public class FullDataSourceV2DTO return dataSource; } + public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException + { return this.createUnitTestDataSource(null); } /** * May be missing one or more data fields.
* Designed to be used without access to Minecraft or any supporting objects. */ - public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException - { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, true); } + public FullDataSourceV2 createUnitTestDataSource(EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException + { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); } - private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException + private FullDataSourceV2 internalPopulateDataSource( + FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, + @Nullable EDhDirection direction, + boolean unitTest) throws IOException, InterruptedException, DataCorruptedException { if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) { @@ -183,10 +204,21 @@ public class FullDataSourceV2DTO throw new DataCorruptedException(e); } - - readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); - readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum); - readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + if (direction == null) + { + readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); + readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum); + readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + } + else + { + // adjacent data is stored in the same byte array + // as the normal data, + // this is done so data sources down-stream + // can all be handled identically regardless of + // whether they're a full or partial data source + readDataSourceAdjacentDataArrayToBlob(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum); + } dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test @@ -231,7 +263,111 @@ public class FullDataSourceV2DTO // (de)serializing // //=================// - private static void writeDataSourceDataArrayToBlob(LongArrayList[] inputDataArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static void writeDataSourceAdjacentDataArrayToBlob( + LongArrayList[] wholeInputDataArray, ByteArrayList outputByteArray, + EDhDirection direction, + EDhApiDataCompressionMode compressionModeEnum) throws IOException + { + // write the outputs to a stream to prep for writing to the database + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + + // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, + // but since this stream will be closed immediately after writing anyway, it won't be an issue + try (DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum)) + { + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + + for (int x = minX; x < maxX; x++) + { + for (int z = minZ; z < maxZ; z++) + { + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList dataColumn = wholeInputDataArray[index]; + + // write column length + short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; + // a short is used instead of an int because at most we store 4096 vertical slices and a + // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) + compressedOut.writeShort(columnLength); + + // write column data (will be skipped if no data was present) + for (int y = 0; y < columnLength; y++) + { + compressedOut.writeLong(dataColumn.getLong(y)); + } + } + } + + + // generate the checksum (currently unused) + compressedOut.flush(); + byteArrayOutputStream.close(); + outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); + } + } + private static void readDataSourceAdjacentDataArrayToBlob( + @NotNull ByteArrayList inputCompressedDataByteArray, @NotNull LongArrayList[] outputDataLongArray, + @NotNull EDhDirection direction, + EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements()); + try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum)) + { + for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++) + { + @NotNull LongArrayList array = outputDataLongArray[i]; + array.clear(); + array.add(FullDataPointUtil.EMPTY_DATA_POINT); + } + + + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + + for (int x = minX; x < maxX; x++) + { + for (int z = minZ; z < maxZ; z++) + { + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList dataColumn = outputDataLongArray[index]; + + // read the column length + short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later + if (dataColumnLength < 0) + { + throw new DataCorruptedException("Read DataSource adj[" + direction + "] Blob data at index [" + index + "], column length [" + dataColumnLength + "] should be greater than zero."); + } + + + ListUtil.clearAndSetSize(dataColumn, dataColumnLength); + + // read column data (will be skipped if no data was present) + for (int y = 0; y < dataColumnLength; y++) + { + long dataPoint = compressedIn.readLong(); + if (VALIDATE_INPUT_DATAPOINTS) + { + FullDataPointUtil.validateDatapoint(dataPoint); + } + dataColumn.set(y, dataPoint); + } + } + } + + } + } + + + private static void writeDataSourceDataArrayToBlob( + LongArrayList[] inputDataArray, ByteArrayList outputByteArray, + EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -266,7 +402,9 @@ public class FullDataSourceV2DTO outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); } } - private static void readBlobToDataSourceDataArray(ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException + private static void readBlobToDataSourceDataArray( + ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, + EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements()); try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum)) @@ -448,7 +586,8 @@ public class FullDataSourceV2DTO // helper methods // //================// - public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } + public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException + { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java new file mode 100644 index 000000000..cc74e7805 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/FullDataMinMaxPosUtil.java @@ -0,0 +1,108 @@ +package com.seibel.distanthorizons.core.sql.dto.util; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; + +/** + * Handles encoding/decoding of min/max X/Z relative {@link FullDataSourceV2#dataPoints} + * positions.
+ * Needed so we can keep the same format between complete data sources + * and incomplete adjacent-only data sources. + */ +public class FullDataMinMaxPosUtil +{ + private static final int ADJ_POS_MASK = (int) Math.pow(2, Short.SIZE) - 1; + private static final int MIN_X_OFFSET = 0; + private static final int MAX_X_OFFSET = Short.SIZE; + private static final int MIN_Z_OFFSET = Short.SIZE * 2; + private static final int MAX_Z_OFFSET = Short.SIZE * 3; + + + + /** + * Encodes min/max X/Z relative {@link FullDataSourceV2#dataPoints} + * positions.
+ * Needed so we can keep the same format between complete data sources + * and incomplete adjacent-only data sources. + */ + public static long getEncodedMinMaxPos(EDhDirection direction) + { + // 4 shorts can fit in a long, and we won't need anything longer than 64 anyway + short minX; + short maxX; + short minZ; + short maxZ; + + switch (direction) + { + case NORTH: + // one row closest to the negative Z axis + minX = 0; + maxX = FullDataSourceV2.WIDTH; + + minZ = 0; + maxZ = 1; + break; + + case SOUTH: + // one row closest to the positive Z axis + minX = 0; + maxX = FullDataSourceV2.WIDTH; + + minZ = FullDataSourceV2.WIDTH - 1; + maxZ = FullDataSourceV2.WIDTH; + break; + + case EAST: + // one row closest to the positive X axis + minX = FullDataSourceV2.WIDTH - 1; + maxX = FullDataSourceV2.WIDTH; + + minZ = 0; + maxZ = FullDataSourceV2.WIDTH; + break; + + case WEST: + // one row closest to the Negative X axis + minX = 0; + maxX = 1; + + minZ = 0; + maxZ = FullDataSourceV2.WIDTH; + break; + + default: + throw new IllegalArgumentException("Unsupported direction [" + direction + "]."); + } + + return encodeAdjMinMaxPos( + minX, maxX, + minZ, maxZ); + } + + public static long encodeAdjMinMaxPos( + short minX, short maxX, + short minZ, short maxZ + ) + { + long data = 0L; + data |= (long) minX << MIN_X_OFFSET; + data |= (long) maxX << MAX_X_OFFSET; + data |= (long) minZ << MIN_Z_OFFSET; + data |= (long) maxZ << MAX_Z_OFFSET; + return data; + } + + public static int getAdjMinX(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MIN_X_OFFSET) & ADJ_POS_MASK); } + public static int getAdjMaxX(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MAX_X_OFFSET) & ADJ_POS_MASK); } + + public static int getAdjMinZ(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MIN_Z_OFFSET) & ADJ_POS_MASK); } + public static int getAdjMaxZ(long encodedMinMaxPos) + { return (int) ((encodedMinMaxPos >> MAX_Z_OFFSET) & ADJ_POS_MASK); } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 6d9f47c5d..7ba0eeaa7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -31,7 +32,6 @@ import com.seibel.distanthorizons.core.util.ListUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; -import com.seibel.distanthorizons.core.logging.DhLogger; import org.jetbrains.annotations.Nullable; import java.io.*; @@ -84,6 +84,9 @@ public class FullDataSourceV2Repo extends AbstractDhRepo= minX && x < maxX + && z >= minZ && z < maxZ) + { + continue; + } + + + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList adjDataColumn = adjSource.dataPoints[index]; + Assert.assertEquals(1, adjDataColumn.size()); + Assert.assertEquals(FullDataPointUtil.EMPTY_DATA_POINT, adjDataColumn.getLong(0)); + } + } + } } diff --git a/core/src/test/java/tests/FullDataMinMaxPosTest.java b/core/src/test/java/tests/FullDataMinMaxPosTest.java new file mode 100644 index 000000000..5a4873f02 --- /dev/null +++ b/core/src/test/java/tests/FullDataMinMaxPosTest.java @@ -0,0 +1,54 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil; +import org.junit.Assert; +import org.junit.Test; + +public class FullDataMinMaxPosTest +{ + + @Test + public void EncodeAdjacentMinMaxPosTest() + { + int maxTest = 3; + for (short minX = 0; minX < maxTest; minX++) + { + for (short maxX = 0; maxX < maxTest; maxX++) + { + for (short minZ = 0; minZ < maxTest; minZ++) + { + for (short maxZ = 0; maxZ < maxTest; maxZ++) + { + long encodedPos = FullDataMinMaxPosUtil.encodeAdjMinMaxPos(minX, maxX, minZ, maxZ); + + Assert.assertEquals(minX, FullDataMinMaxPosUtil.getAdjMinX(encodedPos)); + Assert.assertEquals(maxX, FullDataMinMaxPosUtil.getAdjMaxX(encodedPos)); + Assert.assertEquals(minZ, FullDataMinMaxPosUtil.getAdjMinZ(encodedPos)); + Assert.assertEquals(maxZ, FullDataMinMaxPosUtil.getAdjMaxZ(encodedPos)); + } + } + } + } + + } + +} From 9343854b4aead1088f8bba9a8868711028bb83b0 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 6 Nov 2025 07:42:58 -0600 Subject: [PATCH 08/24] Clean up data source getters --- .../methods/data/DhApiTerrainDataRepo.java | 2 +- .../GeneratedFullDataSourceProvider.java | 2 +- .../RemoteFullDataSourceProvider.java | 8 +-- .../V2/FullDataSourceProviderV2.java | 54 +++++-------------- .../V2/FullDataUpdatePropagatorV2.java | 8 +-- .../fullDatafile/V2/FullDataUpdaterV2.java | 2 +- .../core/generation/PregenManager.java | 2 +- .../server/FullDataSourceRequestHandler.java | 4 +- .../core/render/LodRenderSection.java | 2 +- .../core/sql/repo/FullDataSourceV2Repo.java | 6 --- 10 files changed, 29 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 0dd8f2d15..c27b8a58e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -238,7 +238,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo if (dataSource == null) { // attempt to get/generate the data source for this section - dataSource = level.getFullDataProvider().getAsync(sectionPos, false).get(); + dataSource = level.getFullDataProvider().getAsync(sectionPos).get(); if (dataSource == null) { return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + DhSectionPos.toString(sectionPos) + "]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 47040c4a7..7d21a5898 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -489,7 +489,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im @Override public CompletableFuture shouldGenerateSplitChild(long pos) { - return GeneratedFullDataSourceProvider.this.getAsync(pos, false).thenApply(fullDataSource -> + return GeneratedFullDataSourceProvider.this.getAsync(pos).thenApply(fullDataSource -> { //noinspection TryFinallyCanBeTryWithResources try diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java index 618523718..b3dbae741 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java @@ -75,18 +75,18 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide @Override @Nullable - public FullDataSourceV2 get(long pos, boolean includeAdjacentData) + public FullDataSourceV2 get(long pos) { if (this.syncOnLoadRequestQueue == null) { // we have local data, but networking is unavailable. - return super.get(pos, includeAdjacentData); + return super.get(pos); } if (!this.visitedPositions.add(pos)) { // This position has already been accessed before - return super.get(pos, includeAdjacentData); + return super.get(pos); } @@ -105,7 +105,7 @@ public class RemoteFullDataSourceProvider extends GeneratedFullDataSourceProvide }); } - return super.get(pos, includeAdjacentData); + return super.get(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index d3d83bf23..4487803b4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -171,7 +171,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable * * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ - public CompletableFuture getAsync(long pos, boolean includeAdjacentData) + public CompletableFuture getAsync(long pos) { AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) @@ -182,7 +182,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable try { - return CompletableFuture.supplyAsync(() -> this.get(pos, includeAdjacentData), executor); + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); } catch (RejectedExecutionException ignore) { @@ -193,14 +193,12 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable /** * Should only be used in internal file handler methods where we are already running on a file handler thread. * Can return null if the repo is in the process of being shut down - * @see FullDataSourceProviderV2#getAsync(long, boolean) + * @see FullDataSourceProviderV2#getAsync(long) */ @Nullable - public FullDataSourceV2 get(long pos, boolean includeAdjacentData) + public FullDataSourceV2 get(long pos) { - try(FullDataSourceV2DTO dto = includeAdjacentData - ? this.repo.getByKey(pos) - : this.repo.getByPosNoAdj(pos)) + try(FullDataSourceV2DTO dto = this.repo.getByPosNoAdj(pos)) { if (dto == null) { @@ -245,10 +243,16 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable - // - // TODO name? - // + //=================// + // partial getters // + //=================// + /** + * Only returns the data row/column for the given compass-cardinal + * direction.
+ * This is generally used for generating LOD render data + * where we only need the adjacent data, not the full thing. + */ public FullDataSourceV2 getAdjForDirection(long pos, EDhDirection direction) { try(FullDataSourceV2DTO dto = this.repo.getAdjByPosAndDirection(pos, direction)) @@ -279,36 +283,6 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable return null; } - public FullDataSourceV2 getCenter(long pos) - { - try(FullDataSourceV2DTO dto = this.repo.getByPosNoAdj(pos)) - { - if (dto == null) - { - return FullDataSourceV2.createEmpty(pos); - } - - try - { - // load from database - return this.createDataSourceFromDto(dto); - } - catch (DataCorruptedException e) - { - this.tryLogCorruptedDataError(DhSectionPos.toString(pos), e); - this.repo.deleteWithKey(pos); - } - } - catch (InterruptedException ignore) { } - catch (IOException e) - { - LOGGER.warn("File read Error for pos ["+DhSectionPos.toString(pos)+"], error: "+e.getMessage(), e); - } - - // an error occurred - return null; - } - //=======================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java index f1de0c42e..8f47ac19f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java @@ -183,7 +183,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab parentLocked = true; this.dataUpdater.lockedPosSet.add(parentUpdatePos); - try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos, false)) + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) { // will return null if the file handler is shutting down if (parentDataSource != null) @@ -197,7 +197,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab childReadLock.lock(); this.dataUpdater.lockedPosSet.add(childPos); - try (FullDataSourceV2 childDataSource = this.provider.get(childPos, false)) + try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) { // can return null when the file handler is being shut down if (childDataSource != null) @@ -299,7 +299,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab parentLocked = true; this.dataUpdater.lockedPosSet.add(parentUpdatePos); - try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos, false)) + try (FullDataSourceV2 parentDataSource = this.provider.get(parentUpdatePos)) { // will return null if the file handler is shutting down if (parentDataSource != null) @@ -315,7 +315,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab childWriteLock.lock(); this.dataUpdater.lockedPosSet.add(childPos); - try (FullDataSourceV2 childDataSource = this.provider.get(childPos, false)) + try (FullDataSourceV2 childDataSource = this.provider.get(childPos)) { // will return null if the file handler is shutting down if (childDataSource != null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java index c18a651b2..eefb76c3b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java @@ -122,7 +122,7 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable // get or create the data source - try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos, false)) + try (FullDataSourceV2 recipientDataSource = this.provider.get(updatePos)) { if (recipientDataSource != null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java index 53cabc7ef..044060296 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/PregenManager.java @@ -140,7 +140,7 @@ public class PregenManager } this.pendingGenerations.put(nextSectionPos, System.currentTimeMillis()); - this.fullDataSourceProvider.getAsync(nextSectionPos, false) + this.fullDataSourceProvider.getAsync(nextSectionPos) .thenAccept(fullDataSource -> { if (this.fullDataSourceProvider.isFullyGenerated(fullDataSource.columnGenerationSteps)) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java index 0b0be6c17..2e523ea25 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/multiplayer/server/FullDataSourceRequestHandler.java @@ -99,7 +99,7 @@ public class FullDataSourceRequestHandler } // get the server's datasource - return this.fullDataSourceProvider().get(message.sectionPos, false); + return this.fullDataSourceProvider().get(message.sectionPos); } catch (Exception e) { @@ -262,7 +262,7 @@ public class FullDataSourceRequestHandler private void tryFulfillDataSourceRequestGroup(DataSourceRequestGroup requestGroup, long pos) { - this.fullDataSourceProvider().getAsync(pos, false).thenAccept(fullDataSource -> + this.fullDataSourceProvider().getAsync(pos).thenAccept(fullDataSource -> { if (this.fullDataSourceProvider().isFullyGenerated(fullDataSource.columnGenerationSteps)) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 9732596df..4984bdb7b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -356,7 +356,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable try (FullDataSourceV2 fullDataSource = // no direction means get the center LOD (direction == null) - ? this.fullDataSourceProvider.getCenter(finalPos) + ? this.fullDataSourceProvider.get(finalPos) : this.fullDataSourceProvider.getAdjForDirection(finalPos, direction.opposite())) { getFull.end(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 7ba0eeaa7..5ca9c302c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -356,12 +356,6 @@ public class FullDataSourceV2Repo extends AbstractDhRepo Date: Thu, 6 Nov 2025 21:28:25 -0600 Subject: [PATCH 09/24] Handle non-adjacent data conversion --- .../fullData/sources/FullDataSourceV2.java | 36 ++++++++++- .../V2/FullDataSourceProviderV2.java | 39 +++++++++--- .../core/render/LodQuadTree.java | 3 - .../core/sql/dto/FullDataSourceV2DTO.java | 62 +++++++++++++------ 4 files changed, 108 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index eebb24e92..71f44b254 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataOcclusionCuller; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.file.fullDatafile.V2.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -36,6 +37,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos; +import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil; import com.seibel.distanthorizons.core.util.*; import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; @@ -75,8 +77,6 @@ public class FullDataSourceV2 /** how many chunks wide this datasource is at detail level 0. */ public static final int NUMB_OF_CHUNKS_WIDE = WIDTH / LodUtil.CHUNK_WIDTH; - public static final byte DATA_FORMAT_VERSION = 1; - public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("FullDataV2"); @@ -1094,6 +1094,38 @@ public class FullDataSourceV2 + //===================// + // adjacent clearing // + //===================// + + /** Removes any non-adjacent data from the given direction. */ + public void clearAllNonAdjData(EDhDirection direction) + { + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + + for (int relX = 0; relX < FullDataSourceV2.WIDTH; relX++) + { + for (int relZ = 0; relZ < FullDataSourceV2.WIDTH; relZ++) + { + // skip non-adjacent data + if (relX >= minX && relX < maxX + && relZ >= minZ && relZ < maxZ) + { + continue; + } + + LongArrayList dataColumn = this.getColumnAtRelPos(relX, relZ); + dataColumn.clear(); + dataColumn.add(FullDataPointUtil.EMPTY_DATA_POINT); + } + } + } + + //================// // helper methods // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index 4487803b4..15ff70549 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile.V2; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.enums.EDhDirection; @@ -207,8 +208,19 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable try { - // load from database - return this.createDataSourceFromDto(dto); + FullDataSourceV2 dataSource = this.createDataSourceFromDto(dto); + + // automatically create and save adjacent data if missing + if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA) + { + EDhApiDataCompressionMode compressionMode = Config.Common.LodBuilding.dataCompression.get(); + try(FullDataSourceV2DTO updatedDto = FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionMode)) + { + this.repo.save(updatedDto); + } + } + + return dataSource; } catch (DataCorruptedException e) { @@ -262,6 +274,19 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable return FullDataSourceV2.createEmpty(pos); } + // migrate to the V2 format first if needed + if (dto.dataFormatVersion == FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA) + { + // get automatically converts from V1 to V2 + FullDataSourceV2 migratedDataSource = this.get(pos); + if (migratedDataSource != null) + { + migratedDataSource.clearAllNonAdjData(direction); + } + + return migratedDataSource; + } + try { // load from database @@ -346,14 +371,12 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable - // - // TODO - // + //=============// + // data update // + //=============// public CompletableFuture updateDataSourceAsync(@NotNull FullDataSourceV2 inputData) - { - return this.dataUpdater.updateDataSourceAsync(inputData); - } + { return this.dataUpdater.updateDataSourceAsync(inputData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index fb4cdde72..3fa3dd9e3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -19,8 +19,6 @@ package com.seibel.distanthorizons.core.render; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.LodBufferContainer; @@ -35,7 +33,6 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.render.renderer.generic.BeaconRenderHandler; import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer; -import com.seibel.distanthorizons.core.util.KeyedLockContainer; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.PerfRecorder; import com.seibel.distanthorizons.core.util.ThreadUtil; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 90b79d6c5..2d7917a1b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -54,6 +54,13 @@ public class FullDataSourceV2DTO { public static final boolean VALIDATE_INPUT_DATAPOINTS = true; + public static class DATA_FORMAT + { + public static final int V1_NO_ADJACENT_DATA = 1; + public static final int V2_LATEST = 2; + } + + public long pos; @@ -118,7 +125,7 @@ public class FullDataSourceV2DTO // the mapping hash isn't included since it takes significantly longer to calculate and // as of the time of this comment (2025-1-22) the checksum isn't used for anything so changing it shouldn't cause any issues dto.dataChecksum = dataSource.hashCode(); - dto.dataFormatVersion = FullDataSourceV2.DATA_FORMAT_VERSION; + dto.dataFormatVersion = DATA_FORMAT.V2_LATEST; dto.compressionModeValue = compressionModeEnum.value; dto.lastModifiedUnixDateTime = dataSource.lastModifiedUnixDateTime; dto.createdUnixDateTime = dataSource.createdUnixDateTime; @@ -161,7 +168,7 @@ public class FullDataSourceV2DTO FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(this.pos); try { - this.internalPopulateDataSource(dataSource, levelWrapper, direction, false); + this.populateDataSource(dataSource, levelWrapper, direction, false); } catch (Exception e) { @@ -172,38 +179,57 @@ public class FullDataSourceV2DTO return dataSource; } + /** + * May be missing one or more data fields.
+ * Designed to be used without access to Minecraft. + */ public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException, DataCorruptedException { return this.createUnitTestDataSource(null); } /** * May be missing one or more data fields.
- * Designed to be used without access to Minecraft or any supporting objects. + * Designed to be used without access to Minecraft. */ public FullDataSourceV2 createUnitTestDataSource(EDhDirection direction) throws IOException, InterruptedException, DataCorruptedException - { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); } + { return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), null, direction,true); } - private FullDataSourceV2 internalPopulateDataSource( + private FullDataSourceV2 populateDataSource( FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, @Nullable EDhDirection direction, boolean unitTest) throws IOException, InterruptedException, DataCorruptedException { - if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) + // format validation // + + if (DATA_FORMAT.V1_NO_ADJACENT_DATA != this.dataFormatVersion + && DATA_FORMAT.V2_LATEST != this.dataFormatVersion) { - throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"]."); + throw new IllegalStateException("Data source population only supports formats: ["+DATA_FORMAT.V1_NO_ADJACENT_DATA +","+DATA_FORMAT.V2_LATEST +"], data format found: ["+this.dataFormatVersion+"]."); } + if (direction != null + && this.dataFormatVersion == DATA_FORMAT.V1_NO_ADJACENT_DATA) + { + throw new IllegalStateException("Data format ["+this.dataFormatVersion+"] doesn't support adjacent data. Automatic conversion must be done."); + } + + + + // compression // EDhApiDataCompressionMode compressionModeEnum; try { - compressionModeEnum = this.getCompressionMode(); + compressionModeEnum = EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } catch (IllegalArgumentException e) { - // may happen if ZStd was used (which was added and removed during the nightly builds) - // or if the compressor value is changed to an invalid option + // may happen if the compressor value was changed to an invalid option throw new DataCorruptedException(e); } + + + // data // + if (direction == null) { readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); @@ -220,6 +246,9 @@ public class FullDataSourceV2DTO readDataSourceAdjacentDataArrayToBlob(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum); } + + // mapping // + dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test if (!unitTest) @@ -238,6 +267,10 @@ public class FullDataSourceV2DTO } } + + + // individual properties // + dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; dataSource.createdUnixDateTime = this.createdUnixDateTime; @@ -582,15 +615,6 @@ public class FullDataSourceV2DTO - //================// - // helper methods // - //================// - - public EDhApiDataCompressionMode getCompressionMode() throws IllegalArgumentException - { return EDhApiDataCompressionMode.getFromValue(this.compressionModeValue); } - - - //===========// // overrides // //===========// From 67637dbf10f753f2ef0927fa75dd373054d2bf68 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 6 Nov 2025 21:50:43 -0600 Subject: [PATCH 10/24] detail level renaming --- .../enums/config/EDhApiMaxHorizontalResolution.java | 5 +---- .../fullData/sources/FullDataSourceV2.java | 8 ++++---- .../fullDatafile/V2/FullDataSourceProviderV2.java | 10 +++++----- .../fullDatafile/V2/FullDataUpdatePropagatorV2.java | 2 +- .../distanthorizons/core/render/LodQuadTree.java | 11 +++++------ .../core/util/objects/quadTree/QuadTree.java | 2 +- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java index 65f4399c1..2ecadcc1c 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java @@ -64,10 +64,7 @@ public enum EDhApiMaxHorizontalResolution /** How wide each LOD DataPoint is */ public final int dataPointWidth; - /** - * This is the same as detailLevel in LodQuadTreeNode, - * lowest is 0 highest is 9 - */ + /** This is the same as detailLevel in LodQuadTreeNode */ public final byte detailLevel; /* Start/End X/Z give the block positions diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 71f44b254..b092cff17 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -395,7 +395,7 @@ public class FullDataSourceV2 // copy over application flag if either are set to continue propagating (BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent)) // don't propagate past the top of the tree - && (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.ROOT_SECTION_DETAIL_LEVEL); } // null check to prevent setting a flag we don't want to save in the DB @@ -404,7 +404,7 @@ public class FullDataSourceV2 this.applyToChildren = (BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren)) // don't propagate past the bottom of the tree - && (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.MIN_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.LEAF_SECTION_DETAIL_LEVEL); } } else if (inputDetailLevel + 1 == thisDetailLevel) @@ -415,7 +415,7 @@ public class FullDataSourceV2 this.applyToParent = dataChanged && (BoolUtil.falseIfNull(this.applyToParent) || BoolUtil.falseIfNull(inputDataSource.applyToParent)) - && (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) < FullDataSourceProviderV2.ROOT_SECTION_DETAIL_LEVEL); } else if (inputDetailLevel - 1 == thisDetailLevel) @@ -427,7 +427,7 @@ public class FullDataSourceV2 this.applyToChildren = dataChanged && (BoolUtil.falseIfNull(this.applyToChildren) || BoolUtil.falseIfNull(inputDataSource.applyToChildren)) - && (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.MIN_SECTION_DETAIL_LEVEL); + && (DhSectionPos.getDetailLevel(this.pos) > FullDataSourceProviderV2.LEAF_SECTION_DETAIL_LEVEL); } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index 15ff70549..67c3401ad 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -63,17 +63,17 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable * The highest numerical detail level possible. * Used when determining which positions to update. * - * @see FullDataSourceProviderV2#MIN_SECTION_DETAIL_LEVEL + * @see FullDataSourceProviderV2#LEAF_SECTION_DETAIL_LEVEL */ - public static final byte TOP_SECTION_DETAIL_LEVEL + public static final byte ROOT_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL - + LodUtil.REGION_DETAIL_LEVEL; // TODO how big do we need to go? + + LodUtil.REGION_DETAIL_LEVEL; /** * The lowest numerical detail level possible. * - * @see FullDataSourceProviderV2#TOP_SECTION_DETAIL_LEVEL + * @see FullDataSourceProviderV2#ROOT_SECTION_DETAIL_LEVEL */ - public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + public static final byte LEAF_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java index 8f47ac19f..0d783cdbe 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java @@ -218,7 +218,7 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab } - if (DhSectionPos.getDetailLevel(parentUpdatePos) < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL) + if (DhSectionPos.getDetailLevel(parentUpdatePos) < FullDataSourceProviderV2.ROOT_SECTION_DETAIL_LEVEL) { parentDataSource.applyToParent = true; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 3fa3dd9e3..4191cc995 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -279,9 +279,8 @@ public class LodQuadTree extends QuadTree implements IDebugRen // and disabling render sections // //===============================// - //byte expectedDetailLevel = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + 3; // can be used instead of the following logic for testing byte expectedDetailLevel = this.calculateExpectedDetailLevel(playerPos, sectionPos); - expectedDetailLevel = (byte) Math.min(expectedDetailLevel, this.minRenderDetailLevel); + expectedDetailLevel = (byte) Math.min(expectedDetailLevel, this.minRootRenderDetailLevel); expectedDetailLevel += DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; if (DhSectionPos.getDetailLevel(sectionPos) > expectedDetailLevel) @@ -529,11 +528,11 @@ public class LodQuadTree extends QuadTree implements IDebugRen int detailLevel = (int) (Math.log(distance / this.detailDropOffDistanceUnit) / this.detailDropOffLogBase); - return (byte) MathUtil.clamp(this.maxRenderDetailLevel, detailLevel, Byte.MAX_VALUE - 1); + return (byte) MathUtil.clamp(this.maxLeafRenderDetailLevel, detailLevel, Byte.MAX_VALUE - 1); } private double getDrawDistanceFromDetail(int detail) { - if (detail <= this.maxRenderDetailLevel) + if (detail <= this.maxLeafRenderDetailLevel) { return 0; } @@ -552,14 +551,14 @@ public class LodQuadTree extends QuadTree implements IDebugRen this.detailDropOffDistanceUnit = Config.Client.Advanced.Graphics.Quality.horizontalQuality.get().distanceUnitInBlocks * LodUtil.CHUNK_WIDTH; this.detailDropOffLogBase = Math.log(Config.Client.Advanced.Graphics.Quality.horizontalQuality.get().quadraticBase); - this.maxRenderDetailLevel = Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution.get().detailLevel; + this.maxLeafRenderDetailLevel = Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution.get().detailLevel; // The minimum detail level is done to prevent single corner sections rendering 1 detail level lower than the others. // If not done corners may not be flush with the other LODs, which looks bad. byte minSectionDetailLevel = this.getDetailLevelFromDistance(this.blockRenderDistanceDiameter); // get the minimum allowed detail level minSectionDetailLevel -= 1; // -1 so corners can't render lower than their adjacent neighbors. space minSectionDetailLevel = (byte) Math.min(minSectionDetailLevel, this.treeRootDetailLevel); // don't allow rendering lower detail sections than what the tree contains - this.minRenderDetailLevel = (byte) Math.max(minSectionDetailLevel, this.maxRenderDetailLevel); // respect the user's selected max resolution if it is lower detail (IE they want 2x2 block, but minSectionDetailLevel is specifically for 1x1 block render resolution) + this.minRootRenderDetailLevel = (byte) Math.max(minSectionDetailLevel, this.maxLeafRenderDetailLevel); // respect the user's selected max resolution if it is lower detail (IE they want 2x2 block, but minSectionDetailLevel is specifically for 1x1 block render resolution) } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java index 8812ec36c..c61e3f868 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java @@ -59,7 +59,7 @@ public class QuadTree */ public final byte treeLeafDetailLevel; - private final int diameterInBlocks; // diameterInBlocks + private final int diameterInBlocks; /** contain the actual data in the quad tree structure */ private final MovableGridRingList> topRingList; From 7e04b12e37648d79dbe87171a92db1faf110dcd3 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Fri, 7 Nov 2025 07:41:59 -0600 Subject: [PATCH 11/24] Optimize PrefRecorder slightly --- .../seibel/distanthorizons/core/util/PerfRecorder.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/PerfRecorder.java b/core/src/main/java/com/seibel/distanthorizons/core/util/PerfRecorder.java index d56734b27..3a054a217 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/PerfRecorder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/PerfRecorder.java @@ -175,7 +175,14 @@ public class PerfRecorder long endTime = System.nanoTime(); long totalNano = endTime - this.startTime; - LongAdder nsAdder = PerfRecorder.this.nanoPerId.computeIfAbsent(this.id, (String id) -> new LongAdder()); + LongAdder nsAdder = PerfRecorder.this.nanoPerId.get(this.id); + if (nsAdder != null) + { + nsAdder.add(totalNano); + return; + } + + nsAdder = PerfRecorder.this.nanoPerId.computeIfAbsent(this.id, (String id) -> new LongAdder()); nsAdder.add(totalNano); } } From c374bf7ca8fe9e7ae5cea74405c58d63f1f1ca73 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 08:14:03 -0600 Subject: [PATCH 12/24] test --- .../render/ColumnRenderSource.java | 41 +- .../render/bufferBuilding/ColumnBox.java | 407 ++++++++++-------- .../ColumnRenderBufferBuilder.java | 74 ++-- .../render/bufferBuilding/LodQuadBuilder.java | 17 +- .../FullDataToRenderDataTransformer.java | 119 ++--- .../core/render/LodQuadTree.java | 16 +- .../core/util/RenderDataPointUtil.java | 9 +- 7 files changed, 360 insertions(+), 323 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index af9647c48..ff7badd15 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -23,17 +23,13 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pooling.AbstractPhantomArrayList; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnQuadView; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import it.unimi.dsi.fastutil.longs.LongArrayList; import com.seibel.distanthorizons.core.logging.DhLogger; -import java.util.concurrent.atomic.AtomicLong; - /** * Stores the render data used to generate OpenGL buffers. * @@ -43,10 +39,8 @@ public class ColumnRenderSource extends AbstractPhantomArrayList { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); - public static final boolean DO_SAFETY_CHECKS = ModInfo.IS_DEV_BUILD; - public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - /** width of this data in columns */ - public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); // 64 + /** measured in data columns */ + public static final int WIDTH = 64; public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Render Source"); @@ -63,8 +57,6 @@ public class ColumnRenderSource extends AbstractPhantomArrayList private boolean isEmpty = true; - public AtomicLong localVersion = new AtomicLong(0); // used to track changes to the data source, so that buffers can be updated when necessary - //==============// @@ -88,9 +80,9 @@ public class ColumnRenderSource extends AbstractPhantomArrayList this.verticalDataCount = maxVerticalSize; - this.renderDataContainer = this.pooledArraysCheckout.getLongArray(0, SECTION_SIZE * SECTION_SIZE * this.verticalDataCount); + this.renderDataContainer = this.pooledArraysCheckout.getLongArray(0, WIDTH * WIDTH * this.verticalDataCount); - this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; + this.debugSourceFlags = new DebugSourceFlag[WIDTH * WIDTH]; } @@ -99,19 +91,19 @@ public class ColumnRenderSource extends AbstractPhantomArrayList // datapoint manipulation // //========================// - public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer.getLong(posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex); } + public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer.getLong(posX * WIDTH * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex); } public ColumnArrayView getVerticalDataPointView(int posX, int posZ) { - int offset = posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount; + int offset = posX * WIDTH * this.verticalDataCount + posZ * this.verticalDataCount; // don't allow returning views that are outside this render source's bounds if (offset >= this.renderDataContainer.size()) { return null; } - else if (posX < 0 || posX >= SECTION_SIZE - || posZ < 0 || posZ >= SECTION_SIZE) + else if (posX < 0 || posX >= WIDTH + || posZ < 0 || posZ >= WIDTH) { return null; } @@ -120,8 +112,8 @@ public class ColumnRenderSource extends AbstractPhantomArrayList offset, this.verticalDataCount); } - public ColumnQuadView getFullQuadView() { return this.getQuadViewOverRange(0, 0, SECTION_SIZE, SECTION_SIZE); } - public ColumnQuadView getQuadViewOverRange(int quadX, int quadZ, int quadXSize, int quadZSize) { return new ColumnQuadView(this.renderDataContainer, SECTION_SIZE, this.verticalDataCount, quadX, quadZ, quadXSize, quadZSize); } + public ColumnQuadView getFullQuadView() { return this.getQuadViewOverRange(0, 0, WIDTH, WIDTH); } + public ColumnQuadView getQuadViewOverRange(int quadX, int quadZ, int quadXSize, int quadZSize) { return new ColumnQuadView(this.renderDataContainer, WIDTH, this.verticalDataCount, quadX, quadZ, quadXSize, quadZSize); } @@ -131,9 +123,8 @@ public class ColumnRenderSource extends AbstractPhantomArrayList public Long getPos() { return this.pos; } public Long getKey() { return this.pos; } - public String getKeyDisplayString() { return DhSectionPos.toString(this.pos); } - public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - SECTION_SIZE_OFFSET); } + public byte getDataDetailLevel() { return (byte) (DhSectionPos.getDetailLevel(this.pos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); } public boolean isEmpty() { return this.isEmpty; } public void markNotEmpty() { this.isEmpty = false; } @@ -147,15 +138,15 @@ public class ColumnRenderSource extends AbstractPhantomArrayList } - for (int x = 0; x < SECTION_SIZE; x++) + for (int x = 0; x < WIDTH; x++) { - for (int z = 0; z < SECTION_SIZE; z++) + for (int z = 0; z < WIDTH; z++) { ColumnArrayView columnArrayView = this.getVerticalDataPointView(x,z); for (int i = 0; i < columnArrayView.size; i++) { long dataPoint = columnArrayView.get(i); - if (!RenderDataPointUtil.isVoid(dataPoint)) + if (!RenderDataPointUtil.hasZeroHeight(dataPoint)) { return true; } @@ -179,12 +170,12 @@ public class ColumnRenderSource extends AbstractPhantomArrayList { for (int z = zStart; z < zStart + zWidth; z++) { - this.debugSourceFlags[x * SECTION_SIZE + z] = flag; + this.debugSourceFlags[x * WIDTH + z] = flag; } } } - public DebugSourceFlag debugGetFlag(int ox, int oz) { return this.debugSourceFlags[ox * SECTION_SIZE + oz]; } + public DebugSourceFlag debugGetFlag(int ox, int oz) { return this.debugSourceFlags[ox * WIDTH + oz]; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java index 041c09f1c..aad55a84b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java @@ -23,38 +23,32 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.level.IDhClientLevel; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; +import com.seibel.distanthorizons.core.render.LodQuadTree; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.PerfRecorder; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; -import com.seibel.distanthorizons.core.render.renderer.LodRenderer; import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.jetbrains.annotations.NotNull; +import java.util.ArrayList; import java.util.Arrays; public class ColumnBox { private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); - /** - * if the skylight has this value that means - * no data is expected - */ - private static final byte SKYLIGHT_EMPTY = -1; /** * if the skylight has this value that means - * that block position is covered/occuled by an adjacent block/column. + * that block position is covered/occluded by an adjacent block/column. */ - private static final byte SKYLIGHT_COVERED = -2; + private static final byte SKYLIGHT_COVERED = -1; + + public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Column Box"); - private static final ThreadLocal THREAD_LOCAL_SKY_LIGHT_ARRAY = ThreadLocal.withInitial(() -> - { - byte[] array = new byte[RenderDataPointUtil.MAX_WORLD_Y_SIZE]; - Arrays.fill(array, SKYLIGHT_EMPTY); - return array; - }); @@ -64,7 +58,7 @@ public class ColumnBox public static void addBoxQuadsToBuilder( LodQuadBuilder builder, IDhClientLevel clientLevel, - short xSize, short ySize, short zSize, + short width, short yHeight, short minX, short minY, short minZ, int color, byte irisBlockMaterialId, byte skyLight, byte blockLight, long topData, long bottomData, ColumnArrayView[] adjData, boolean[] isAdjDataSameDetailLevel) @@ -73,9 +67,9 @@ public class ColumnBox // variable setup // //================// - short maxX = (short) (minX + xSize); - short maxY = (short) (minY + ySize); - short maxZ = (short) (minZ + zSize); + short maxX = (short) (minX + width); + short maxY = (short) (minY + yHeight); + short maxZ = (short) (minZ + width); byte skyLightTop = skyLight; byte skyLightBot = RenderDataPointUtil.doesDataPointExist(bottomData) ? RenderDataPointUtil.getLightSky(bottomData) : 0; @@ -111,15 +105,15 @@ public class ColumnBox if (!isTransparent && isTopTransparent && RenderDataPointUtil.doesDataPointExist(topData)) { skyLightTop = (byte) MathUtil.clamp(0, 15 - (RenderDataPointUtil.getYMax(topData) - minY), 15); - ySize = (short) (RenderDataPointUtil.getYMax(topData) - minY - 1); + yHeight = (short) (RenderDataPointUtil.getYMax(topData) - minY - 1); } else if (isTransparent && !isBottomTransparent && RenderDataPointUtil.doesDataPointExist(bottomData)) { - minY = (short) (minY + ySize - 1); - ySize = 1; + minY = (short) (minY + yHeight - 1); + yHeight = 1; } - maxY = (short) (minY + ySize); + maxY = (short) (minY + yHeight); } @@ -128,16 +122,20 @@ public class ColumnBox // add top and bottom faces // //==========================// - boolean skipTop = RenderDataPointUtil.doesDataPointExist(topData) && (RenderDataPointUtil.getYMin(topData) == maxY) && !isTopTransparent; + boolean skipTop = RenderDataPointUtil.doesDataPointExist(topData) + && (RenderDataPointUtil.getYMin(topData) == maxY) + && !isTopTransparent; if (!skipTop) { - builder.addQuadUp(minX, maxY, minZ, xSize, zSize, ColorUtil.applyShade(color, MC.getShade(EDhDirection.UP)), irisBlockMaterialId, skyLightTop, blockLight); + builder.addQuadUp(minX, maxY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.UP)), irisBlockMaterialId, skyLightTop, blockLight); } - boolean skipBottom = RenderDataPointUtil.doesDataPointExist(bottomData) && (RenderDataPointUtil.getYMax(bottomData) == minY) && !isBottomTransparent; + boolean skipBottom = RenderDataPointUtil.doesDataPointExist(bottomData) + && (RenderDataPointUtil.getYMax(bottomData) == minY) + && !isBottomTransparent; if (!skipBottom) { - builder.addQuadDown(minX, minY, minZ, xSize, zSize, ColorUtil.applyShade(color, MC.getShade(EDhDirection.DOWN)), irisBlockMaterialId, skyLightBot, blockLight); + builder.addQuadDown(minX, minY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.DOWN)), irisBlockMaterialId, skyLightBot, blockLight); } @@ -156,12 +154,12 @@ public class ColumnBox // Add an adjacent face if this is opaque face or transparent over the void. if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.NORTH, minX, minY, minZ, xSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj(EDhDirection.NORTH, minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.NORTH, minX, minY, minZ, xSize, ySize, + makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.NORTH, minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } @@ -174,12 +172,12 @@ public class ColumnBox { if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.SOUTH, minX, minY, maxZ, xSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj(EDhDirection.SOUTH, minX, minY, maxZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.SOUTH, minX, minY, maxZ, xSize, ySize, + makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.SOUTH, minX, minY, maxZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } @@ -192,12 +190,12 @@ public class ColumnBox { if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.WEST, minX, minY, minZ, zSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj(EDhDirection.WEST, minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.WEST, minX, minY, minZ, zSize, ySize, + makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.WEST, minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } @@ -210,19 +208,19 @@ public class ColumnBox { if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.EAST, maxX, minY, minZ, zSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj(EDhDirection.EAST, maxX, minY, minZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.EAST, maxX, minY, minZ, zSize, ySize, + makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.EAST, maxX, minY, minZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } } private static void makeAdjVerticalQuad( - LodQuadBuilder builder, @NotNull ColumnArrayView adjColumnView, boolean adjacentIsSameDetailLevel, int caveCullingMaxY, EDhDirection direction, + LodQuadBuilder builder, @NotNull ColumnArrayView adjColumnView, boolean adjacentIsSameDetailLevel, int caveCullingMaxY, EDhDirection direction, short x, short yMin, short z, short horizontalWidth, short ySize, int color, byte irisBlockMaterialId, byte blockLight) { @@ -233,11 +231,9 @@ public class ColumnBox color = ColorUtil.applyShade(color, MC.getShade(direction)); - // if there isn't any data adjacent to this LOD, - // just add the full vertical quad - if (adjColumnView.size == 0 || RenderDataPointUtil.isVoid(adjColumnView.get(0))) + if (adjColumnView.size == 0 + || RenderDataPointUtil.hasZeroHeight(adjColumnView.get(0))) { - builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); return; } @@ -245,165 +241,139 @@ public class ColumnBox //===========================// - // Determine face visibility // - // based on it's neighbors // + // Build Y-range segments // + // with their sky light // //===========================// boolean transparencyEnabled = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled; + boolean inputTransparent = ColorUtil.getAlpha(color) < 255 && transparencyEnabled; + short yMax = (short) (yMin + ySize); - short yMax = (short) (yMin + ySize); // min is inclusive, max is exclusive - byte[] skyLightAtInputPos = THREAD_LOCAL_SKY_LIGHT_ARRAY.get(); + // List to store segments: [startY, endY, skyLight] + ArrayList segments = new ArrayList<>(); - try + int adjCount = adjColumnView.size(); + + // Start with the entire range at max light + segments.add(new YSegment(yMin, yMax, LodUtil.MAX_MC_LIGHT)); + + // Process each adjacent datapoint and split/update segments + for (int adjIndex = 0; adjIndex < adjCount; adjIndex++) { - // set the initial sky-lights for this face, - // if nothing overlaps or overhangs the face should have max sky light - Arrays.fill(skyLightAtInputPos, yMin, yMax, LodUtil.MAX_MC_LIGHT); + long adjPoint = adjColumnView.get(adjIndex); + short adjMinY = RenderDataPointUtil.getYMin(adjPoint); + short adjMaxY = RenderDataPointUtil.getYMax(adjPoint); - // iterate top down - int adjCount = adjColumnView.size(); - for (int adjIndex = 0; adjIndex < adjCount; adjIndex++) + if (!RenderDataPointUtil.doesDataPointExist(adjPoint) + || RenderDataPointUtil.hasZeroHeight(adjPoint) + || yMax <= adjMinY) { - long adjPoint = adjColumnView.get(adjIndex); - short adjMinY = RenderDataPointUtil.getYMin(adjPoint); - short adjMaxY = RenderDataPointUtil.getYMax(adjPoint); - - // skip empty adjacent datapoints - if (!RenderDataPointUtil.doesDataPointExist(adjPoint) - || RenderDataPointUtil.isVoid(adjPoint)) - { - continue; - } - - // skip this adjacent datapoint if it's above the input datapoint (since it can't affect the input data point) - if (yMax <= adjMinY) - { - continue; - } - - - long adjAbovePoint = (adjIndex != 0) ? adjColumnView.get(adjIndex - 1) : RenderDataPointUtil.EMPTY_DATA; - long adjBelowPoint = (adjIndex + 1 < adjCount) ? adjColumnView.get(adjIndex + 1) : RenderDataPointUtil.EMPTY_DATA; - - // if the adjacent data point is over the void - // don't consider it as transparent - boolean adjOverVoid = !RenderDataPointUtil.doesDataPointExist(adjBelowPoint); - boolean adjTransparent = !adjOverVoid - && RenderDataPointUtil.getAlpha(adjPoint) < 255 - && transparencyEnabled; - - - - //=================================// - // set sky light based on adjacent // - //=================================// - - // set light based on overlapping adjacent - if (!adjTransparent) - { - // adj opaque - // mark positions adjacent is covering - byte adjSkyLight = RenderDataPointUtil.getLightSky(adjPoint); - for (int i = adjMinY; i < adjMaxY; i++) - { - byte skyLightAtPos = skyLightAtInputPos[i]; - - // if the adjacent is a different detail level, we want to render adjacent opaque - // faces to try and reduce the chance of holes on detail level borders - boolean adjacentCoversThis = - // if the adjacent is the same detail level, no special handling is necessary - !adjacentIsSameDetailLevel - // if the adjacent face is underground we probably don't need it - && RenderDataPointUtil.getYMax(adjPoint) >= caveCullingMaxY - // check if this face is on a border - && - ( - (x == 0 && direction == EDhDirection.WEST) - || (z == 0 && direction == EDhDirection.NORTH) - // TODO why does 256 represent a border? aren't LODs only 64 datapoints wide? - || (x == 256 && direction == EDhDirection.EAST) - || (z == 256 && direction == EDhDirection.SOUTH) - ); - - byte newSkyLightAtPos = adjacentCoversThis ? adjSkyLight : SKYLIGHT_COVERED; - skyLightAtInputPos[i] = (byte) Math.min(newSkyLightAtPos, skyLightAtPos); - } - } - else - { - // adjacent is transparent, - // use datapoint below adjacent for lighting - byte belowSkyLight = RenderDataPointUtil.getLightSky(adjBelowPoint); - for (int i = adjMinY; i < adjMaxY; i++) - { - byte skyLightAtPos = skyLightAtInputPos[i]; - skyLightAtInputPos[i] = (byte) Math.min(belowSkyLight, skyLightAtPos); - } - } - - - // fill in sky light up to the next DP, - // this is done to handle overhangs - byte adjSkyLight = RenderDataPointUtil.getLightSky(adjPoint); - int adjAboveMinY = RenderDataPointUtil.getYMin(adjAbovePoint); - for (int i = adjMaxY; i < adjAboveMinY; i++) - { - byte skyLightAtPos = skyLightAtInputPos[i]; - skyLightAtInputPos[i] = (byte) Math.min(adjSkyLight, skyLightAtPos); - } + continue; } + long adjAbovePoint = (adjIndex != 0) ? adjColumnView.get(adjIndex - 1) : RenderDataPointUtil.EMPTY_DATA; + long adjBelowPoint = (adjIndex + 1 < adjCount) ? adjColumnView.get(adjIndex + 1) : RenderDataPointUtil.EMPTY_DATA; + boolean adjOverVoid = !RenderDataPointUtil.doesDataPointExist(adjBelowPoint); + boolean adjTransparent = !adjOverVoid + && RenderDataPointUtil.getAlpha(adjPoint) < 255 + && transparencyEnabled; - //=======================// - // create vertical faces // - //=======================// + byte adjSkyLight = RenderDataPointUtil.getLightSky(adjPoint); + byte lightToApply; - boolean inputTransparent = ColorUtil.getAlpha(color) < 255 && transparencyEnabled; - byte lastSkyLight = skyLightAtInputPos[yMin]; - int quadBottomY = yMin; - int quadTopY = -1; - - // walk up the sky lights and create a new face - // whenever the light changes to different valid value - for (int i = yMin; i < yMax; i++) + if (!adjTransparent) { - byte skyLight = skyLightAtInputPos[i]; - if (skyLight != lastSkyLight) - { - // the sky light changed, create the in-progress face - tryAddVerticalFaceWithSkyLightToBuilder( - builder, direction, - x, z, horizontalWidth, - color, irisBlockMaterialId, blockLight, - lastSkyLight, inputTransparent, quadTopY, quadBottomY + // Adjacent is opaque + boolean adjacentCoversThis = + !adjacentIsSameDetailLevel + && RenderDataPointUtil.getYMax(adjPoint) >= caveCullingMaxY + && + ( + (x == 0 && direction == EDhDirection.WEST) + || (z == 0 && direction == EDhDirection.NORTH) + || (x == 256 && direction == EDhDirection.EAST) + || (z == 256 && direction == EDhDirection.SOUTH) ); - - lastSkyLight = skyLight; - quadBottomY = i; - } - quadTopY = (i + 1); + lightToApply = adjacentCoversThis ? adjSkyLight : SKYLIGHT_COVERED; + } + else + { + // Adjacent is transparent, use below light + lightToApply = RenderDataPointUtil.getLightSky(adjBelowPoint); } - // add the in-progress face if present - if (quadTopY != -1) + + // Apply light to the range [adjMinY, adjMaxY) + applyLightToRange(segments, adjMinY, adjMaxY, lightToApply); + + // Fill overhang area [adjMaxY, adjAboveMinY) with adjSkyLight + int adjAboveMinY = RenderDataPointUtil.getYMin(adjAbovePoint); + if (adjMaxY < adjAboveMinY) { - tryAddVerticalFaceWithSkyLightToBuilder( - builder, direction, - x, z, horizontalWidth, - color, irisBlockMaterialId, blockLight, - lastSkyLight, inputTransparent, quadTopY, quadBottomY - ); + applyLightToRange(segments, adjMaxY, adjAboveMinY, adjSkyLight); } } - finally + + + + //=======================// + // Create vertical faces // + // from segments // + //=======================// + + for (YSegment seg : segments) { - // clean up the array before the next thread uses it - // (may be unnecessary since we only work between the yMin-yMax anyway, but is helpful for debugging) - Arrays.fill(skyLightAtInputPos, yMin, yMax, SKYLIGHT_EMPTY); + tryAddVerticalFaceWithSkyLightToBuilder( + builder, direction, + x, z, horizontalWidth, + color, irisBlockMaterialId, blockLight, + seg.skyLight, inputTransparent, seg.endY, seg.startY + ); } } + + // Apply a light value to a Y range, splitting segments as needed + private static void applyLightToRange(ArrayList segments, int rangeStart, int rangeEnd, byte newLight) + { + ArrayList newSegments = new ArrayList<>(); + + for (YSegment seg : segments) + { + // No overlap + if (seg.endY <= rangeStart + || seg.startY >= rangeEnd) + { + newSegments.add(seg); + continue; + } + + // Partial or complete overlap - need to split + + // Part before the range + if (seg.startY < rangeStart) + { + newSegments.add(new YSegment(seg.startY, rangeStart, seg.skyLight)); + } + + // Overlapping part - take minimum light + int overlapStart = Math.max(seg.startY, rangeStart); + int overlapEnd = Math.min(seg.endY, rangeEnd); + byte minLight = (byte) Math.min(newLight, seg.skyLight); + newSegments.add(new YSegment(overlapStart, overlapEnd, minLight)); + + // Part after the range + if (seg.endY > rangeEnd) + { + newSegments.add(new YSegment(rangeEnd, seg.endY, seg.skyLight)); + } + } + + segments.clear(); + segments.addAll(newSegments); + } + private static void tryAddVerticalFaceWithSkyLightToBuilder( LodQuadBuilder builder, EDhDirection direction, short x, short z, short horizontalWidth, @@ -412,24 +382,85 @@ public class ColumnBox ) { // invalid positions will have a negative skylight - if (lastSkyLight >= 0) + if (lastSkyLight < 0) { - // Don't add transparent vertical faces - // unless the adjacent position is empty. - // This is done to prevent walls between water blocks in the ocean. - if (!inputTransparent - || (lastSkyLight == LodUtil.MAX_MC_LIGHT)) - { - // don't add negative/empty height faces - short height = (short) (quadTopY - quadBottomY); - if (height > 0) - { - builder.addQuadAdj(direction, x, (short) quadBottomY, z, horizontalWidth, height, color, irisBlockMaterialId, lastSkyLight, blockLight); - } - } + return; } + + // Don't add transparent vertical faces + // unless the adjacent position is empty. + // This is done to prevent walls between water blocks in the ocean. + if (inputTransparent + && (lastSkyLight != LodUtil.MAX_MC_LIGHT)) + { + return; + } + + // don't add negative/empty height faces + short height = (short) (quadTopY - quadBottomY); + if (height <= 0) + { + return; + } + + builder.addQuadAdj( + direction, + x, (short) quadBottomY, z, + horizontalWidth, height, + color, irisBlockMaterialId, lastSkyLight, blockLight); } + private static class YSegment + { + int startY; + int endY; + byte skyLight; + + YSegment(int startY, int endY, byte skyLight) + { + this.startY = startY; + this.endY = endY; + this.skyLight = skyLight; + } + } + + + /** + * @see com.seibel.distanthorizons.core.util.FullDataPointUtil + */ + private static class YSegmentUtil + { + private static final int HEIGHT_WIDTH = Short.SIZE; + private static final int SKY_LIGHT_WIDTH = Byte.SIZE; + + private static final int START_Y_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1; + private static final int END_Y_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1; + private static final int SKY_LIGHT_MASK = (int) Math.pow(2, SKY_LIGHT_WIDTH) - 1; + + private static final int START_Y_OFFSET = 0; + private static final int END_Y_OFFSET = START_Y_OFFSET + HEIGHT_WIDTH; + private static final int SKY_LIGHT_OFFSET = END_Y_OFFSET + HEIGHT_WIDTH; + + + + public static long encode(short startY, short endY, byte skyLight) + { + long data = 0L; + data |= (long) (startY & START_Y_MASK) << START_Y_OFFSET; + data |= (long) (endY & END_Y_MASK) << END_Y_OFFSET; + data |= (long) (skyLight & SKY_LIGHT_MASK) << SKY_LIGHT_OFFSET; + return data; + } + + public static short getStartY(long data) { return (short) ((data >> START_Y_OFFSET) & START_Y_MASK); } + public static short getEndY(long data) { return (short) ((data >> END_Y_OFFSET) & END_Y_MASK); } + public static short getSkyLight(long data) { return (short) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } + + } + + + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index a75b1566d..7d8cd325f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -33,7 +33,6 @@ import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; -import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; @@ -106,18 +105,15 @@ public class ColumnRenderBufferBuilder //===================// byte thisDetailLevel = renderSource.getDataDetailLevel(); - for (int relX = 0; relX < ColumnRenderSource.SECTION_SIZE; relX++) + for (int relX = 0; relX < ColumnRenderSource.WIDTH; relX++) { - for (int relZ = 0; relZ < ColumnRenderSource.SECTION_SIZE; relZ++) + for (int relZ = 0; relZ < ColumnRenderSource.WIDTH; relZ++) { - // stop the builder if requested - UncheckedInterruptedException.throwIfInterrupted(); - // ignore empty/null columns ColumnArrayView columnRenderData = renderSource.getVerticalDataPointView(relX, relZ); if (columnRenderData.size() == 0 - || !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0)) - || RenderDataPointUtil.isVoid(columnRenderData.get(0))) + || !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0)) + || RenderDataPointUtil.hasZeroHeight(columnRenderData.get(0))) { continue; } @@ -158,8 +154,8 @@ public class ColumnRenderBufferBuilder int xAdj = relX + lodDirection.normal.x; int zAdj = relZ + lodDirection.normal.z; boolean isCrossRenderSourceBoundary = - (xAdj < 0 || xAdj >= ColumnRenderSource.SECTION_SIZE) || - (zAdj < 0 || zAdj >= ColumnRenderSource.SECTION_SIZE); + (xAdj < 0 || xAdj >= ColumnRenderSource.WIDTH) || + (zAdj < 0 || zAdj >= ColumnRenderSource.WIDTH); ColumnRenderSource adjRenderSource; byte adjDetailLevel; @@ -196,20 +192,20 @@ public class ColumnRenderBufferBuilder if (xAdj < 0) { - xAdj += ColumnRenderSource.SECTION_SIZE; + xAdj += ColumnRenderSource.WIDTH; } - if (xAdj >= ColumnRenderSource.SECTION_SIZE) + if (xAdj >= ColumnRenderSource.WIDTH) { - xAdj -= ColumnRenderSource.SECTION_SIZE; + xAdj -= ColumnRenderSource.WIDTH; } if (zAdj < 0) { - zAdj += ColumnRenderSource.SECTION_SIZE; + zAdj += ColumnRenderSource.WIDTH; } - if (zAdj >= ColumnRenderSource.SECTION_SIZE) + if (zAdj >= ColumnRenderSource.WIDTH) { - zAdj -= ColumnRenderSource.SECTION_SIZE; + zAdj -= ColumnRenderSource.WIDTH; } } } @@ -244,15 +240,14 @@ public class ColumnRenderBufferBuilder ColumnRenderSource.DebugSourceFlag debugSourceFlag = renderSource.debugGetFlag(relX, relZ); - // We render every vertical lod present in this position - // We only stop when we find a block that is void or non-existing block for (int i = 0; i < columnRenderData.size(); i++) { // can be uncommented to limit which vertical LOD is generated if (Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugEnable.get()) { int wantedColumnIndex = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugColumnIndex.get(); - if (wantedColumnIndex >= 0 && i != wantedColumnIndex) + if (wantedColumnIndex >= 0 + && i != wantedColumnIndex) { continue; } @@ -261,7 +256,8 @@ public class ColumnRenderBufferBuilder long data = columnRenderData.get(i); // If the data is not render-able (Void or non-existing) we stop since there is // no data left in this position - if (RenderDataPointUtil.isVoid(data) || !RenderDataPointUtil.doesDataPointExist(data)) + if (RenderDataPointUtil.hasZeroHeight(data) + || !RenderDataPointUtil.doesDataPointExist(data)) { break; } @@ -269,7 +265,7 @@ public class ColumnRenderBufferBuilder long topDataPoint = (i - 1) >= 0 ? columnRenderData.get(i - 1) : RenderDataPointUtil.EMPTY_DATA; long bottomDataPoint = (i + 1) < columnRenderData.size() ? columnRenderData.get(i + 1) : RenderDataPointUtil.EMPTY_DATA; - addLodToBuffer( + addRenderDataPointToBuilder( clientLevel, data, topDataPoint, bottomDataPoint, adjColumnViews, isSameDetailLevel, @@ -282,31 +278,31 @@ public class ColumnRenderBufferBuilder quadBuilder.mergeQuads(); } - private static void addLodToBuffer( + private static void addRenderDataPointToBuilder( IDhClientLevel clientLevel, - long data, long topData, long bottomData, + long renderData, long topRenderData, long bottomRenderData, ColumnArrayView[] adjColumnViews, boolean[] isSameDetailLevel, byte detailLevel, int renderSourceOffsetPosX, int renderSourceOffsetPosZ, LodQuadBuilder quadBuilder, ColumnRenderSource.DebugSourceFlag debugSource) { long sectionPos = DhSectionPos.encode(detailLevel, renderSourceOffsetPosX, renderSourceOffsetPosZ); - short width = (short) BitShiftUtil.powerOfTwo(detailLevel); - short xMin = (short) DhSectionPos.getMinCornerBlockX(sectionPos); - short yMin = RenderDataPointUtil.getYMin(data); - short zMin = (short) DhSectionPos.getMinCornerBlockZ(sectionPos); - short ySize = (short) (RenderDataPointUtil.getYMax(data) - yMin); + short blockWidth = (short) DhSectionPos.getDetailLevelWidthInBlocks(detailLevel); + short blockMinX = (short) DhSectionPos.getMinCornerBlockX(sectionPos); + short blockMinY = RenderDataPointUtil.getYMin(renderData); + short blockMinZ = (short) DhSectionPos.getMinCornerBlockZ(sectionPos); + short blockMaxY = (short) (RenderDataPointUtil.getYMax(renderData) - blockMinY); - if (ySize == 0) + if (blockMaxY == 0) { return; } - else if (ySize < 0) + else if (blockMaxY < 0) { - throw new IllegalArgumentException("Negative y size for the data! Data: [" + RenderDataPointUtil.toString(data) + "]."); + throw new IllegalArgumentException("Negative y size for the renderDataPoint! Data: [" + RenderDataPointUtil.toString(renderData) + "]."); } - byte blockMaterialId = RenderDataPointUtil.getBlockMaterialId(data); + byte blockMaterialId = RenderDataPointUtil.getBlockMaterialId(renderData); @@ -321,11 +317,11 @@ public class ColumnRenderBufferBuilder float brightnessMultiplier = Config.Client.Advanced.Graphics.Quality.brightnessMultiplier.get().floatValue(); if (saturationMultiplier == 1.0 && brightnessMultiplier == 1.0) { - color = RenderDataPointUtil.getColor(data); + color = RenderDataPointUtil.getColor(renderData); } else { - float[] ahsv = ColorUtil.argbToAhsv(RenderDataPointUtil.getColor(data)); + float[] ahsv = ColorUtil.argbToAhsv(RenderDataPointUtil.getColor(renderData)); color = ColorUtil.ahsvToArgb(ahsv[0], ahsv[1], ahsv[2] * saturationMultiplier, ahsv[3] * brightnessMultiplier); } break; @@ -416,13 +412,13 @@ public class ColumnRenderBufferBuilder ColumnBox.addBoxQuadsToBuilder( quadBuilder, clientLevel, - width, ySize, width, - xMin, yMin, zMin, + blockWidth, blockMaxY, + blockMinX, blockMinY, blockMinZ, color, blockMaterialId, - RenderDataPointUtil.getLightSky(data), - fullBright ? 15 : RenderDataPointUtil.getLightBlock(data), - topData, bottomData, adjColumnViews, isSameDetailLevel); + RenderDataPointUtil.getLightSky(renderData), + fullBright ? LodUtil.MAX_MC_LIGHT : RenderDataPointUtil.getLightBlock(renderData), + topRenderData, bottomRenderData, adjColumnViews, isSameDetailLevel); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index 463e171b8..2ea274322 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -138,7 +138,8 @@ public class LodQuadBuilder //===========// public void addQuadAdj( - EDhDirection dir, short x, short y, short z, + EDhDirection dir, + short x, short y, short z, short widthEastWest, short widthNorthSouthOrUpDown, int color, byte irisBlockMaterialId, byte skyLight, byte blockLight) { @@ -149,11 +150,11 @@ public class LodQuadBuilder BufferQuad quad = new BufferQuad(x, y, z, widthEastWest, widthNorthSouthOrUpDown, color, irisBlockMaterialId, skyLight, blockLight, dir); ArrayList quadList = (this.doTransparency && ColorUtil.getAlpha(color) < 255) ? this.transparentQuads[dir.ordinal()] : this.opaqueQuads[dir.ordinal()]; - if (!quadList.isEmpty() && - ( - quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.EastWest) - || quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.NorthSouthOrUpDown)) - ) + if (!quadList.isEmpty() + && ( + quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.EastWest) + || quadList.get(quadList.size() - 1).tryMerge(quad, BufferMergeDirectionEnum.NorthSouthOrUpDown)) + ) { this.premergeCount++; return; @@ -174,8 +175,8 @@ public class LodQuadBuilder public void addQuadDown(short x, short y, short z, short width, short wz, int color, byte irisBlockMaterialId, byte skylight, byte blocklight) { BufferQuad quad = new BufferQuad(x, y, z, width, wz, color, irisBlockMaterialId, skylight, blocklight, EDhDirection.DOWN); - ArrayList qs = (doTransparency && ColorUtil.getAlpha(color) < 255) - ? transparentQuads[EDhDirection.DOWN.ordinal()] : opaqueQuads[EDhDirection.DOWN.ordinal()]; + ArrayList qs = (this.doTransparency && ColorUtil.getAlpha(color) < 255) + ? this.transparentQuads[EDhDirection.DOWN.ordinal()] : this.opaqueQuads[EDhDirection.DOWN.ordinal()]; qs.add(quad); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 52fa961e0..cb4100455 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -31,6 +31,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPosMutable; +import com.seibel.distanthorizons.core.render.LodQuadTree; import com.seibel.distanthorizons.core.util.*; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -138,7 +139,7 @@ public class FullDataToRenderDataTransformer } } - columnSource.fillDebugFlag(0, 0, ColumnRenderSource.SECTION_SIZE, ColumnRenderSource.SECTION_SIZE, ColumnRenderSource.DebugSourceFlag.FULL); + columnSource.fillDebugFlag(0, 0, ColumnRenderSource.WIDTH, ColumnRenderSource.WIDTH, ColumnRenderSource.DebugSourceFlag.FULL); return columnSource; } @@ -173,7 +174,10 @@ public class FullDataToRenderDataTransformer // expand the ColumnArrayView to fit the new larger max vertical size ColumnArrayView newColumnArrayView = new ColumnArrayView(dataArrayList, fullDataLength, 0, fullDataLength); setRenderColumnView(levelWrapper, fullDataSource, blockX, blockZ, newColumnArrayView, fullDataColumn); + + PerfRecorder.Timer vertSize = LodQuadTree.TRANSFORM_PERF_RECORDER.start("vertSize"); columnArrayView.changeVerticalSizeFrom(newColumnArrayView); + vertSize.end(); } finally { @@ -186,6 +190,8 @@ public class FullDataToRenderDataTransformer int blockX, int blockZ, ColumnArrayView renderColumnData, LongArrayList fullColumnData) { + PerfRecorder.Timer prep = LodQuadTree.TRANSFORM_PERF_RECORDER.start("prep"); + //===============// // config values // //===============// @@ -217,6 +223,8 @@ public class FullDataToRenderDataTransformer int blocklightToApplyToNextBlock = -1; int renderDataIndex = 0; + prep.end(); + //==================================// @@ -230,6 +238,8 @@ public class FullDataToRenderDataTransformer // goes from the top down for (int fullDataIndex = 0; fullDataIndex < fullColumnData.size(); fullDataIndex++) { + PerfRecorder.Timer fullParse = LodQuadTree.TRANSFORM_PERF_RECORDER.start("fullParse"); + long fullData = fullColumnData.getLong(fullDataIndex); int bottomY = FullDataPointUtil.getBottomY(fullData); @@ -265,6 +275,8 @@ public class FullDataToRenderDataTransformer continue; } + fullParse.end(); + //====================// @@ -272,48 +284,57 @@ public class FullDataToRenderDataTransformer // cave culling check // //====================// - boolean ignoreBlock = blockStatesToIgnore.contains(block); - boolean caveBlock = caveBlockStatesToIgnore.contains(block); // TODO caves should also ignore transparent/non-solid blocks (IE grass and plants) wthout each being defined - if (caveBlock) + PerfRecorder.Timer caveCull = LodQuadTree.TRANSFORM_PERF_RECORDER.start("caveCull"); + + try { - if (caveCullingEnabled - // assume this data point is underground if it has no sky-light - && skyLight == LodUtil.MIN_MC_LIGHT - // ignore caves above a certain height to prevent floating islands from having walls underneath them - && topY < caveCullingMaxY - // cave culling shouldn't happen when at the top of the world - && renderDataIndex != 0 && fullDataIndex != 0 - // cave culling can't happen when at the bottom of the world - && (fullDataIndex+1) < fullColumnData.size()) + boolean ignoreBlock = blockStatesToIgnore.contains(block); + boolean caveBlock = caveBlockStatesToIgnore.contains(block); // TODO caves should also ignore transparent/non-solid blocks (IE grass and plants) wthout each being defined + if (caveBlock) { - // we need to get the next sky/block lights because - // the air block here will always have a light of 0/0 due to only the top of the LOD's light being saved. - long nextFullData = fullColumnData.getLong(fullDataIndex+1); - int nextSkyLight = FullDataPointUtil.getSkyLight(nextFullData); - - if (nextSkyLight == LodUtil.MIN_MC_LIGHT - && ColorUtil.getAlpha(lastColor) == 255) + if (caveCullingEnabled + // assume this data point is underground if it has no sky-light + && skyLight == LodUtil.MIN_MC_LIGHT + // ignore caves above a certain height to prevent floating islands from having walls underneath them + && topY < caveCullingMaxY + // cave culling shouldn't happen when at the top of the world + && renderDataIndex != 0 && fullDataIndex != 0 + // cave culling can't happen when at the bottom of the world + && (fullDataIndex + 1) < fullColumnData.size()) { - // replace the previous block with new bottom - long columnData = renderColumnData.get(renderDataIndex - 1); - columnData = RenderDataPointUtil.setYMin(columnData, bottomY); - renderColumnData.set(renderDataIndex - 1, columnData); + // we need to get the next sky/block lights because + // the air block here will always have a light of 0/0 due to only the top of the LOD's light being saved. + long nextFullData = fullColumnData.getLong(fullDataIndex + 1); + int nextSkyLight = FullDataPointUtil.getSkyLight(nextFullData); + + if (nextSkyLight == LodUtil.MIN_MC_LIGHT + && ColorUtil.getAlpha(lastColor) == 255) + { + // replace the previous block with new bottom + long columnData = renderColumnData.get(renderDataIndex - 1); + columnData = RenderDataPointUtil.setYMin(columnData, bottomY); + renderColumnData.set(renderDataIndex - 1, columnData); + } + + continue; } - continue; + + if (ignoreBlock) + { + // this is a merged block and a cave block, so it should never be rendered + continue; + } } - - - if (ignoreBlock) + else if (ignoreBlock) { - // this is a merged block and a cave block, so it should never be rendered + // this is an ignored block, but shouldn't be merged like a cave block continue; } } - else if (ignoreBlock) + finally { - // this is an ignored block, but shouldn't be merged like a cave block - continue; + caveCull.end(); } @@ -322,20 +343,22 @@ public class FullDataToRenderDataTransformer // non-solid block check // //=======================// - if (ignoreNonCollidingBlocks - && !block.isSolid() - && !block.isLiquid() - && block.getOpacity() != LodUtil.BLOCK_FULLY_OPAQUE) + if (ignoreNonCollidingBlocks + && !block.isSolid() + && !block.isLiquid() + && block.getOpacity() != LodUtil.BLOCK_FULLY_OPAQUE) { if (colorBelowWithAvoidedBlocks) { + PerfRecorder.Timer nonSolid = LodQuadTree.TRANSFORM_PERF_RECORDER.start("color-NonSolid"); int tempColor = levelWrapper.getBlockColor(mutableBlockPos, biome, fullDataSource, block); + nonSolid.end(); // don't transfer the color when alpha is 0 // this prevents issues if grass is transparent if (ColorUtil.getAlpha(tempColor) != 0) { - colorToApplyToNextBlock = ColorUtil.setAlpha(tempColor,255); + colorToApplyToNextBlock = ColorUtil.setAlpha(tempColor, 255); skylightToApplyToNextBlock = skyLight; blocklightToApplyToNextBlock = blockLight; } @@ -349,8 +372,10 @@ public class FullDataToRenderDataTransformer int color; if (colorToApplyToNextBlock == -1) { + PerfRecorder.Timer colorTimer = LodQuadTree.TRANSFORM_PERF_RECORDER.start("color"); // use this block's color color = levelWrapper.getBlockColor(mutableBlockPos, biome, fullDataSource, block); + colorTimer.end(); } else { @@ -366,6 +391,7 @@ public class FullDataToRenderDataTransformer //=============================// // merge same-colored adjacent // //=============================// + PerfRecorder.Timer mergeSame = LodQuadTree.TRANSFORM_PERF_RECORDER.start("mergeSame"); // check if they share a top-bottom face and if they have same color if (color == lastColor @@ -387,6 +413,8 @@ public class FullDataToRenderDataTransformer } lastBottom = bottomY; lastColor = color; + + mergeSame.end(); } @@ -398,21 +426,4 @@ public class FullDataToRenderDataTransformer - //================// - // helper methods // - //================// - - /** - * Called in loops that may run for an extended period of time.
- * This is necessary to allow canceling these transformers since running - * them after the client has left a given world will throw exceptions. - */ - private static void throwIfThreadInterrupted() throws InterruptedException - { - if (Thread.interrupted()) - { - throw new InterruptedException(FullDataToRenderDataTransformer.class.getSimpleName() + " task interrupted."); - } - } - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 4191cc995..3fed7c1a5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -59,8 +59,6 @@ import java.util.concurrent.locks.ReentrantLock; */ public class LodQuadTree extends QuadTree implements IDebugRenderable, AutoCloseable { - public static final byte TREE_LOWEST_DETAIL_LEVEL = ColumnRenderSource.SECTION_SIZE_OFFSET; - private static final DhLogger LOGGER = new DhLoggerBuilder().build(); /** there should only ever be one {@link LodQuadTree} so having the thread static should be fine */ private static final ThreadPoolExecutor FULL_DATA_RETRIEVAL_QUEUE_THREAD = ThreadUtil.makeSingleThreadPool("QuadTree Full Data Retrieval Queue Populator"); @@ -99,11 +97,13 @@ public class LodQuadTree extends QuadTree implements IDebugRen // TODO should be removed once James is done testing @Deprecated public static final PerfRecorder FILE_PERF_RECORDER = new PerfRecorder("File"); + @Deprecated + public static final PerfRecorder TRANSFORM_PERF_RECORDER = new PerfRecorder("Transform"); /** the smallest numerical detail level number that can be rendered */ - private byte maxRenderDetailLevel; + private byte maxLeafRenderDetailLevel; /** the largest numerical detail level number that can be rendered */ - private byte minRenderDetailLevel; + private byte minRootRenderDetailLevel; /** used to calculate when a detail drop will occur */ private double detailDropOffDistanceUnit; @@ -121,7 +121,7 @@ public class LodQuadTree extends QuadTree implements IDebugRen int initialPlayerBlockX, int initialPlayerBlockZ, FullDataSourceProviderV2 fullDataSourceProvider) { - super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL); + super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showQuadTreeRenderStatus); @@ -133,6 +133,8 @@ public class LodQuadTree extends QuadTree implements IDebugRen this.beaconRenderHandler = (genericObjectRenderer != null) ? new BeaconRenderHandler(genericObjectRenderer) : null; FILE_PERF_RECORDER.clear(); + TRANSFORM_PERF_RECORDER.clear(); + COL_BOX_PERF_RECORDER.clear(); } @@ -157,6 +159,10 @@ public class LodQuadTree extends QuadTree implements IDebugRen } + FILE_PERF_RECORDER.tryLog(); + TRANSFORM_PERF_RECORDER.tryLog(); + COL_BOX_PERF_RECORDER.tryLog(); + // this shouldn't be updated while the tree is being iterated through this.updateDetailLevelVariables(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index 59504ed39..b7dc0ec37 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -24,6 +24,7 @@ import com.seibel.distanthorizons.core.level.AbstractDhLevel; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.logging.DhLogger; @@ -202,9 +203,9 @@ public class RenderDataPointUtil return dataPoint & ~(HEIGHT_SHIFTED_MASK | DEPTH_SHIFTED_MASK) | height | depth; } - /** AKA the ending/top/highest Y value above {@link AbstractDhLevel#getMinY()} */ + /** AKA the ending/top/highest Y value above {@link ILevelWrapper#getMinHeight()} ()} */ public static short getYMax(long dataPoint) { return (short) ((dataPoint >>> HEIGHT_SHIFT) & HEIGHT_MASK); } - /** AKA the starting/bottom/lowest Y value above {@link AbstractDhLevel#getMinY()} */ + /** AKA the starting/bottom/lowest Y value above {@link ILevelWrapper#getMinHeight()} */ public static short getYMin(long dataPoint) { return (short) ((dataPoint >>> DEPTH_SHIFT) & DEPTH_MASK); } public static long setYMin(long dataPoint, int depth) { return (long) ((dataPoint & ~(DEPTH_MASK << DEPTH_SHIFT)) | (depth & DEPTH_MASK) << DEPTH_SHIFT); } @@ -219,7 +220,7 @@ public class RenderDataPointUtil public static byte getBlockMaterialId(long dataPoint) { return (byte) ((dataPoint >>> IRIS_BLOCK_MATERIAL_ID_SHIFT) & IRIS_BLOCK_MATERIAL_ID_MASK); } - public static boolean isVoid(long dataPoint) { return (((dataPoint >>> DEPTH_SHIFT) & HEIGHT_DEPTH_MASK) == 0); } + public static boolean hasZeroHeight(long dataPoint) { return (((dataPoint >>> DEPTH_SHIFT) & HEIGHT_DEPTH_MASK) == 0); } public static boolean doesDataPointExist(long dataPoint) { return dataPoint != EMPTY_DATA; } @@ -240,7 +241,7 @@ public class RenderDataPointUtil { return "null"; } - else if (isVoid(dataPoint)) + else if (hasZeroHeight(dataPoint)) { return "void"; } From 27fb629c22bf112b008e96398fc231041f3b0797 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 17:41:07 -0600 Subject: [PATCH 13/24] default unsafe UI values to config option --- .../com/seibel/distanthorizons/core/config/ConfigHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java index 65d555c25..32f0e58fb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java @@ -98,7 +98,7 @@ public class ConfigHandler * Disables the minimum and maximum validation.
* Fun to use, but should be disabled by default. */ - public boolean runMinMaxValidation = true; + public boolean runMinMaxValidation = Config.Client.Advanced.Debugging.allowUnsafeValues.get(); From 5c5d39738e4207d209319eed266d1a003a8f3f16 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 17:44:08 -0600 Subject: [PATCH 14/24] minor reformating --- .../render/bufferBuilding/LodQuadBuilder.java | 15 ++++++++++----- .../render/columnViews/ColumnArrayView.java | 4 +--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index 2ea274322..936b10f65 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -166,18 +166,23 @@ public class LodQuadBuilder // XZ public void addQuadUp(short minX, short maxY, short minZ, short widthEastWest, short widthNorthSouthOrUpDown, int color, byte irisBlockMaterialId, byte skylight, byte blocklight) // TODO argument names are wrong { - BufferQuad quad = new BufferQuad(minX, maxY, minZ, widthEastWest, widthNorthSouthOrUpDown, color, irisBlockMaterialId, skylight, blocklight, EDhDirection.UP); boolean isTransparent = (this.doTransparency && ColorUtil.getAlpha(color) < 255); - ArrayList quadList = isTransparent ? this.transparentQuads[EDhDirection.UP.ordinal()] : this.opaqueQuads[EDhDirection.UP.ordinal()]; + ArrayList quadList = isTransparent + ? this.transparentQuads[EDhDirection.UP.ordinal()] + : this.opaqueQuads[EDhDirection.UP.ordinal()]; + + BufferQuad quad = new BufferQuad(minX, maxY, minZ, widthEastWest, widthNorthSouthOrUpDown, color, irisBlockMaterialId, skylight, blocklight, EDhDirection.UP); quadList.add(quad); } public void addQuadDown(short x, short y, short z, short width, short wz, int color, byte irisBlockMaterialId, byte skylight, byte blocklight) { + ArrayList quadArray = (this.doTransparency && ColorUtil.getAlpha(color) < 255) + ? this.transparentQuads[EDhDirection.DOWN.ordinal()] + : this.opaqueQuads[EDhDirection.DOWN.ordinal()]; + BufferQuad quad = new BufferQuad(x, y, z, width, wz, color, irisBlockMaterialId, skylight, blocklight, EDhDirection.DOWN); - ArrayList qs = (this.doTransparency && ColorUtil.getAlpha(color) < 255) - ? this.transparentQuads[EDhDirection.DOWN.ordinal()] : this.opaqueQuads[EDhDirection.DOWN.ordinal()]; - qs.add(quad); + quadArray.add(quad); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java index f8b35ece8..9c5a630fb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java @@ -101,9 +101,7 @@ public final class ColumnArrayView implements IColumnDataView @Override public ColumnArrayView subView(int dataIndexStart, int dataCount) - { - return new ColumnArrayView(data, dataCount * verticalSize, offset + dataIndexStart * verticalSize, verticalSize); - } + { return new ColumnArrayView(data, dataCount * verticalSize, offset + dataIndexStart * verticalSize, verticalSize); } public void fill(long value) { Arrays.fill(data.elements(), offset, offset + size, value); } From f9dfc38bf16486570045a7608f950a11671b6350 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 17:47:50 -0600 Subject: [PATCH 15/24] Separate BlockBiomeWrapperPair from FullDataPointIdMap --- .../core/config/file/ConfigFileHandler.java | 6 +- .../dataObjects/BlockBiomeWrapperPair.java | 151 +++++++++++ .../fullData/FullDataPointIdMap.java | 242 ++++-------------- 3 files changed, 206 insertions(+), 193 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandler.java index e4f0b2512..4b46f0a07 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/file/ConfigFileHandler.java @@ -240,7 +240,7 @@ public class ConfigFileHandler else if (entry.getTrueValue() == null) { // TODO when can this happen? - throw new IllegalArgumentException("Entry [" + entry.getNameAndCategory() + "] is null, this may be a problem with [" + ModInfo.NAME + "]. Please contact the authors."); + throw new IllegalArgumentException("BlockBiomeWrapperPair [" + entry.getNameAndCategory() + "] is null, this may be a problem with [" + ModInfo.NAME + "]. Please contact the authors."); } workConfig.set(entry.getNameAndCategory(), ConfigTypeConverters.attemptToConvertToString(entry.getType(), entry.getTrueValue())); @@ -287,13 +287,13 @@ public class ConfigFileHandler if (entry.getTrueValue() == null) { - LOGGER.warn("Entry [" + entry.getNameAndCategory() + "] returned as null from the config. Using default value."); + LOGGER.warn("BlockBiomeWrapperPair [" + entry.getNameAndCategory() + "] returned as null from the config. Using default value."); entry.setWithoutFiringEvents(entry.getDefaultValue()); } } catch (Exception e) { - LOGGER.warn("Entry [" + entry.getNameAndCategory() + "] had an invalid value when loading the config. Using default value."); + LOGGER.warn("BlockBiomeWrapperPair [" + entry.getNameAndCategory() + "] had an invalid value when loading the config. Using default value."); entry.setWithoutFiringEvents(entry.getDefaultValue()); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java new file mode 100644 index 000000000..3ec642291 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/BlockBiomeWrapperPair.java @@ -0,0 +1,151 @@ +package com.seibel.distanthorizons.core.dataObjects; + +import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; +import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; +import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * A pooled compound key between the biome and blockState.
+ * These objects are pooled since we will need this compound key + * many times. + * + * @see FullDataPointIdMap + * @see IBlockStateWrapper + * @see IBiomeWrapper + */ +public class BlockBiomeWrapperPair +{ + private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class); + + /** two levels are present so we don't need to use a key object */ + private static final ConcurrentHashMap> CACHED_PAIR_BY_BIOME_BY_BLOCK = new ConcurrentHashMap<>(); + + public final IBiomeWrapper biome; + public final IBlockStateWrapper blockState; + + private int hashCode = 0; + private boolean hashGenerated = false; + private String serialString = null; + + + + //=============// + // constructor // + //=============// + + public static BlockBiomeWrapperPair get(IBlockStateWrapper blockState, IBiomeWrapper biome) + { + // check for existing entry + ConcurrentHashMap pairByBiomeWrapper = CACHED_PAIR_BY_BIOME_BY_BLOCK.get(blockState); + if (pairByBiomeWrapper != null) + { + BlockBiomeWrapperPair pair = pairByBiomeWrapper.get(biome); + if (pair != null) + { + return pair; + } + } + + // Lazily create the inner map and new BlockBiomeWrapperPair + return CACHED_PAIR_BY_BIOME_BY_BLOCK + .computeIfAbsent(blockState, newBlockState -> new ConcurrentHashMap<>()) + .computeIfAbsent(biome, newBiome -> new BlockBiomeWrapperPair(biome, blockState)); + } + private BlockBiomeWrapperPair(IBiomeWrapper biome, IBlockStateWrapper blockState) + { + this.biome = biome; + this.blockState = blockState; + } + + + + //===========// + // overrides // + //===========// + + /** + * Reminder: this hash code won't always be unique, collisions can occur; + * because of that this hash shouldn't be the only unique identifier for this object. + */ + @Override + public int hashCode() + { + // cache the hash code to improve speed + if (!this.hashGenerated) + { + this.hashCode = generateHashCode(this); + this.hashGenerated = true; + } + + return this.hashCode; + } + private static int generateHashCode(BlockBiomeWrapperPair pair) { return generateHashCode(pair.biome, pair.blockState); } + private static int generateHashCode(IBiomeWrapper biome, IBlockStateWrapper blockState) + { + final int prime = 31; + + int result = 1; + // the biome and blockstate hashcode should be already calculated by the time + // we get here, so this operation should be very fast + result = prime * result + (biome == null ? 0 : biome.hashCode()); + result = prime * result + (blockState == null ? 0 : blockState.hashCode()); + return result; + } + + @Override + public boolean equals(Object otherObj) + { + if (otherObj == this) + { + return true; + } + + if (!(otherObj instanceof BlockBiomeWrapperPair)) + { + return false; + } + + BlockBiomeWrapperPair other = (BlockBiomeWrapperPair) otherObj; + return other.biome.getSerialString().equals(this.biome.getSerialString()) + && other.blockState.getSerialString().equals(this.blockState.getSerialString()); + } + + @Override + public String toString() { return this.serialize(); } + + + + //=================// + // (de)serializing // + //=================// + + public String serialize() + { + if (this.serialString == null) + { + this.serialString = this.biome.getSerialString() + FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString(); + } + + return this.serialString; + } + + public static BlockBiomeWrapperPair deserialize(String str, ILevelWrapper levelWrapper) throws DataCorruptedException + { + int separatorIndex = str.indexOf(FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING); + if (separatorIndex == -1) + { + throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry ["+str+"], unable to find separator."); + } + + IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapperOrGetDefault(str.substring(0, separatorIndex), levelWrapper); + IBlockStateWrapper blockState = WRAPPER_FACTORY.deserializeBlockStateWrapperOrGetDefault(str.substring(separatorIndex+FullDataPointIdMap.BLOCK_STATE_SEPARATOR_STRING.length()), levelWrapper); + return BlockBiomeWrapperPair.get(blockState, biome); + } + +} \ No newline at end of file diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index b9034ded9..de96c6c97 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData; -import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; +import com.seibel.distanthorizons.core.dataObjects.BlockBiomeWrapperPair; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.LodUtil; @@ -28,9 +28,7 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStrea import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; -import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import org.apache.logging.log4j.LogManager; import com.seibel.distanthorizons.core.logging.DhLogger; import java.io.*; @@ -60,15 +58,15 @@ public class FullDataPointIdMap */ private static final boolean RUN_SERIALIZATION_DUPLICATE_VALIDATION = false; /** Distant Horizons - Block State Wrapper */ - private static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_"; + public static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_"; /** should only be used for debugging */ private long pos; - /** The index should be the same as the Entry's ID */ - private final ArrayList entryList = new ArrayList<>(); - private final ConcurrentHashMap idMap = new ConcurrentHashMap<>(); + /** The index should be the same as the BlockBiomeWrapperPair's ID */ + private final ArrayList blockBiomePairList = new ArrayList<>(); + private final ConcurrentHashMap idMap = new ConcurrentHashMap<>(); private int cachedHashCode = 0; @@ -90,28 +88,28 @@ public class FullDataPointIdMap public IBiomeWrapper getBiomeWrapper(int id) throws IndexOutOfBoundsException { return this.getEntry(id).biome; } /** @see FullDataPointIdMap#getEntry(int) */ public IBlockStateWrapper getBlockStateWrapper(int id) throws IndexOutOfBoundsException { return this.getEntry(id).blockState; } - /** @throws IndexOutOfBoundsException if the given ID isn't in the {@link FullDataPointIdMap#entryList} */ - private Entry getEntry(int id) throws IndexOutOfBoundsException + /** @throws IndexOutOfBoundsException if the given ID isn't in the {@link FullDataPointIdMap#blockBiomePairList} */ + private BlockBiomeWrapperPair getEntry(int id) throws IndexOutOfBoundsException { - Entry entry; + BlockBiomeWrapperPair pair; try { - entry = this.entryList.get(id); + pair = this.blockBiomePairList.get(id); } catch (IndexOutOfBoundsException e) { - throw new IndexOutOfBoundsException("FullData ID Map out of sync for pos: "+DhSectionPos.toString(this.pos)+". ID: ["+id+"] greater than the number of known ID's: ["+this.entryList.size()+"]."); + throw new IndexOutOfBoundsException("FullData ID Map out of sync for pos: "+DhSectionPos.toString(this.pos)+". ID: ["+id+"] greater than the number of known ID's: ["+this.blockBiomePairList.size()+"]."); } - return entry; + return pair; } /** @return -1 if the list is empty */ - public int getMaxValidId() { return this.entryList.size() - 1; } - public int size() { return this.entryList.size(); } + public int getMaxValidId() { return this.blockBiomePairList.size() - 1; } + public int size() { return this.blockBiomePairList.size(); } - public boolean isEmpty() { return this.entryList.isEmpty(); } + public boolean isEmpty() { return this.blockBiomePairList.isEmpty(); } public long getPos() { return this.pos; } @@ -125,11 +123,11 @@ public class FullDataPointIdMap * If an entry with the given values already exists nothing will * be added but the existing item's ID will still be returned. */ - public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(Entry.getEntry(biome, blockState)); } - private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry) + public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(BlockBiomeWrapperPair.get(blockState, biome)); } + private int addIfNotPresentAndGetId(BlockBiomeWrapperPair pair) { // try getting the existing ID - Integer nullableId = this.idMap.get(biomeBlockStateEntry); + Integer nullableId = this.idMap.get(pair); if (nullableId != null) { return nullableId; @@ -137,7 +135,7 @@ public class FullDataPointIdMap // create the new ID - return this.idMap.compute(biomeBlockStateEntry, (Entry newBiomeBlockStateEntry, Integer currentId) -> + return this.idMap.compute(pair, (BlockBiomeWrapperPair newPair, Integer currentId) -> { if (currentId != null) { @@ -146,8 +144,8 @@ public class FullDataPointIdMap // Add the new ID - currentId = this.entryList.size(); - this.entryList.add(biomeBlockStateEntry); + currentId = this.blockBiomePairList.size(); + this.blockBiomePairList.add(newPair); // invalidate the cached hash code this.cachedHashCode = 0; @@ -157,7 +155,7 @@ public class FullDataPointIdMap } /** - * Adds every {@link Entry} from inputMap into this map.
+ * Adds every {@link BlockBiomeWrapperPair} from inputMap into this map.
* Allows duplicate entries.

* * Allowing duplicate entries should be done if a datasource is just being read in and @@ -167,19 +165,19 @@ public class FullDataPointIdMap */ public void addAll(FullDataPointIdMap inputMap) { - ArrayList entriesToMerge = inputMap.entryList; - for (int i = 0; i < entriesToMerge.size(); i++) + ArrayList pairsToMerge = inputMap.blockBiomePairList; + for (int i = 0; i < pairsToMerge.size(); i++) { - Entry entity = entriesToMerge.get(i); - this.add(entity); + BlockBiomeWrapperPair pair = pairsToMerge.get(i); + this.add(pair); } } - /** allows for adding duplicate {@link Entry} */ - private void add(Entry biomeBlockStateEntry) + /** allows for adding duplicate {@link BlockBiomeWrapperPair} */ + private void add(BlockBiomeWrapperPair pair) { - int id = this.entryList.size(); - this.entryList.add(biomeBlockStateEntry); - this.idMap.put(biomeBlockStateEntry, id); + int id = this.blockBiomePairList.size(); + this.blockBiomePairList.add(pair); + this.idMap.put(pair, id); // invalidate the cached hash code this.cachedHashCode = 0; @@ -196,23 +194,23 @@ public class FullDataPointIdMap */ public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap inputMap) { - ArrayList entriesToMerge = inputMap.entryList; - int[] remappedEntryIds = new int[entriesToMerge.size()]; + ArrayList entriesToMerge = inputMap.blockBiomePairList; + int[] remappedPairIds = new int[entriesToMerge.size()]; for (int i = 0; i < entriesToMerge.size(); i++) { - Entry entity = entriesToMerge.get(i); + BlockBiomeWrapperPair entity = entriesToMerge.get(i); int id = this.addIfNotPresentAndGetId(entity); - remappedEntryIds[i] = id; + remappedPairIds[i] = id; } - return remappedEntryIds; + return remappedPairIds; } /** Should only be used if this map is going to be reused, otherwise bad things will happen. */ public void clear(long pos) { this.pos = pos; - this.entryList.clear(); + this.blockBiomePairList.clear(); this.idMap.clear(); this.cachedHashCode = 0; } @@ -226,27 +224,27 @@ public class FullDataPointIdMap /** Serializes all contained entries into the given stream, formatted in UTF */ public void serialize(DhDataOutputStream outputStream) throws IOException { - outputStream.writeInt(this.entryList.size()); + outputStream.writeInt(this.blockBiomePairList.size()); // only used when debugging - HashMap dataPointEntryBySerialization = new HashMap<>(); + HashMap dataPointEntryBySerialization = new HashMap<>(); - for (Entry entry : this.entryList) + for (BlockBiomeWrapperPair pair : this.blockBiomePairList) { - String entryString = entry.serialize(); + String entryString = pair.serialize(); outputStream.writeUTF(entryString); if (RUN_SERIALIZATION_DUPLICATE_VALIDATION) { if (dataPointEntryBySerialization.containsKey(entryString)) { - LOGGER.error("Duplicate serialized entry found with serial: " + entryString); + LOGGER.error("Duplicate serialized pair found with serial: " + entryString); } - if (dataPointEntryBySerialization.containsValue(entry)) + if (dataPointEntryBySerialization.containsValue(pair)) { - LOGGER.error("Duplicate serialized entry found with value: " + entry.serialize()); + LOGGER.error("Duplicate serialized pair found with value: " + pair.serialize()); } - dataPointEntryBySerialization.put(entryString, entry); + dataPointEntryBySerialization.put(entryString, pair); } } } @@ -262,7 +260,7 @@ public class FullDataPointIdMap // only used when debugging - HashMap dataPointEntryBySerialization = new HashMap<>(); + HashMap dataPointEntryBySerialization = new HashMap<>(); FullDataPointIdMap newMap = new FullDataPointIdMap(pos); for (int i = 0; i < entityCount; i++) @@ -275,8 +273,8 @@ public class FullDataPointIdMap String entryString = inputStream.readUTF(); - Entry newEntry = Entry.deserialize(entryString, levelWrapper); - newMap.entryList.add(newEntry); + BlockBiomeWrapperPair newPair = BlockBiomeWrapperPair.deserialize(entryString, levelWrapper); + newMap.blockBiomePairList.add(newPair); if (RUN_SERIALIZATION_DUPLICATE_VALIDATION) { @@ -284,11 +282,11 @@ public class FullDataPointIdMap { LOGGER.error("Duplicate deserialized entry found with serial: " + entryString); } - if (dataPointEntryBySerialization.containsValue(newEntry)) + if (dataPointEntryBySerialization.containsValue(newPair)) { - LOGGER.error("Duplicate deserialized entry found with value: " + newEntry.serialize()); + LOGGER.error("Duplicate deserialized entry found with value: " + newPair.serialize()); } - dataPointEntryBySerialization.put(entryString, newEntry); + dataPointEntryBySerialization.put(entryString, newPair); } } @@ -334,149 +332,13 @@ public class FullDataPointIdMap private void generateHashCode() { int result = DhSectionPos.hashCode(this.pos); - for (int i = 0; i < this.entryList.size(); i++) + for (int i = 0; i < this.blockBiomePairList.size(); i++) { - result = 31 * result + this.entryList.hashCode(); + result = 31 * result + this.blockBiomePairList.hashCode(); } this.cachedHashCode = result; } - //==============// - // helper class // - //==============// - - private static final class Entry - { - private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class); - - /** two levels are present so we don't need to use a key object */ - private static final ConcurrentHashMap> ENTRY_BY_BLOCKSTATE_BY_BIOMEWRAPPER = new ConcurrentHashMap<>(); - - public final IBiomeWrapper biome; - public final IBlockStateWrapper blockState; - - private int hashCode = 0; - private boolean hashGenerated = false; - private String serialString = null; - - - - //=============// - // constructor // - //=============// - - public static Entry getEntry(IBiomeWrapper biome, IBlockStateWrapper blockState) - { - // check for existing entry - ConcurrentHashMap entryByBlockState = ENTRY_BY_BLOCKSTATE_BY_BIOMEWRAPPER.get(biome); - if (entryByBlockState != null) - { - Entry entry = entryByBlockState.get(blockState); - if (entry != null) - { - return entry; - } - } - - // Lazily create the inner map and new Entry - return ENTRY_BY_BLOCKSTATE_BY_BIOMEWRAPPER - .computeIfAbsent(biome, newBiome -> new ConcurrentHashMap<>()) - .computeIfAbsent(blockState, newBlockState -> new Entry(biome, blockState)); - } - private Entry(IBiomeWrapper biome, IBlockStateWrapper blockState) - { - this.biome = biome; - this.blockState = blockState; - } - - - - //===========// - // overrides // - //===========// - - /** - * Reminder: this hash code won't always be unique, collisions can occur; - * because of that this hash shouldn't be the only unique identifier for this object. - */ - @Override - public int hashCode() - { - // cache the hash code to improve speed - if (!this.hashGenerated) - { - this.hashCode = generateHashCode(this); - this.hashGenerated = true; - } - - return this.hashCode; - } - private static int generateHashCode(Entry entry) { return generateHashCode(entry.biome, entry.blockState); } - private static int generateHashCode(IBiomeWrapper biome, IBlockStateWrapper blockState) - { - final int prime = 31; - - int result = 1; - // the biome and blockstate hashcode should be already calculated by the time - // we get here, so this operation should be very fast - result = prime * result + (biome == null ? 0 : biome.hashCode()); - result = prime * result + (blockState == null ? 0 : blockState.hashCode()); - return result; - } - - @Override - public boolean equals(Object otherObj) - { - if (otherObj == this) - { - return true; - } - - if (!(otherObj instanceof Entry)) - { - return false; - } - - Entry other = (Entry) otherObj; - return other.biome.getSerialString().equals(this.biome.getSerialString()) - && other.blockState.getSerialString().equals(this.blockState.getSerialString()); - } - - @Override - public String toString() { return this.serialize(); } - - - - //=================// - // (de)serializing // - //=================// - - public String serialize() - { - if (this.serialString == null) - { - this.serialString = this.biome.getSerialString() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString(); - } - - return this.serialString; - } - - public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws DataCorruptedException - { - int separatorIndex = str.indexOf(BLOCK_STATE_SEPARATOR_STRING); - if (separatorIndex == -1) - { - throw new DataCorruptedException("Failed to deserialize BiomeBlockStateEntry ["+str+"], unable to find separator."); - } - - IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapperOrGetDefault(str.substring(0, separatorIndex), levelWrapper); - IBlockStateWrapper blockState = WRAPPER_FACTORY.deserializeBlockStateWrapperOrGetDefault(str.substring(separatorIndex+BLOCK_STATE_SEPARATOR_STRING.length()), levelWrapper); - return Entry.getEntry(biome, blockState); - } - - } - - } From f0acc73c5640e967e7af688d2c5d143e0140d16b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 17:48:30 -0600 Subject: [PATCH 16/24] Add compass Index to Edirection --- .../core/enums/EDhDirection.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java b/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java index 337a1b544..12902c7fa 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/enums/EDhDirection.java @@ -32,17 +32,17 @@ import com.seibel.distanthorizons.core.util.math.Vec3i; public enum EDhDirection { /** negative Y */ - DOWN("down", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Y, new Vec3i(0, -1, 0)), + DOWN("down", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Y, new Vec3i(0, -1, 0), -1), /** positive Y */ - UP("up", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Y, new Vec3i(0, 1, 0)), + UP("up", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Y, new Vec3i(0, 1, 0), -1), /** negative Z */ - NORTH("north", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, -1)), + NORTH("north", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, -1), 0), /** positive Z */ - SOUTH("south", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, 1)), + SOUTH("south", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.Z, new Vec3i(0, 0, 1), 1), /** negative X */ - WEST("west", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.X, new Vec3i(-1, 0, 0)), + WEST("west", EDhDirection.AxisDirection.NEGATIVE, EDhDirection.Axis.X, new Vec3i(-1, 0, 0), 2), /** positive X */ - EAST("east", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.X, new Vec3i(1, 0, 0)); + EAST("east", EDhDirection.AxisDirection.POSITIVE, EDhDirection.Axis.X, new Vec3i(1, 0, 0), 3); /** Up, Down, West, East, North, South */ @@ -69,6 +69,8 @@ public enum EDhDirection public final EDhDirection.Axis axis; public final EDhDirection.AxisDirection axisDirection; public final Vec3i normal; + /** -1 if not a {@link EDhDirection#CARDINAL_COMPASS} direction */ + public final int compassIndex; @@ -76,12 +78,13 @@ public enum EDhDirection // constructor // //=============// - EDhDirection(String name, EDhDirection.AxisDirection axisDirection, EDhDirection.Axis axis, Vec3i normal) + EDhDirection(String name, EDhDirection.AxisDirection axisDirection, EDhDirection.Axis axis, Vec3i normal, int compassIndex) { this.name = name; this.axis = axis; this.axisDirection = axisDirection; this.normal = normal; + this.compassIndex = compassIndex; } From b5199cfa87454f01eaa1209157edecffa03695f2 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 18:08:02 -0600 Subject: [PATCH 17/24] Optimize ColumnBox building --- .../render/bufferBuilding/ColumnBox.java | 200 ++++++----- .../ColumnRenderBufferBuilder.java | 321 +++++++++--------- .../FullDataToRenderDataTransformer.java | 88 ++--- .../core/render/LodQuadTree.java | 6 - .../core/render/LodRenderSection.java | 16 +- 5 files changed, 323 insertions(+), 308 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java index aad55a84b..77c76360b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java @@ -23,20 +23,17 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.level.IDhClientLevel; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; -import com.seibel.distanthorizons.core.render.LodQuadTree; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.PerfRecorder; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.coreapi.util.MathUtil; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.jetbrains.annotations.NotNull; -import java.util.ArrayList; -import java.util.Arrays; - public class ColumnBox { private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); @@ -47,8 +44,6 @@ public class ColumnBox */ private static final byte SKYLIGHT_COVERED = -1; - public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Column Box"); - @@ -57,7 +52,7 @@ public class ColumnBox //=========// public static void addBoxQuadsToBuilder( - LodQuadBuilder builder, IDhClientLevel clientLevel, + LodQuadBuilder builder, PhantomArrayListCheckout phantomArrayCheckout, IDhClientLevel clientLevel, short width, short yHeight, short minX, short minY, short minZ, int color, byte irisBlockMaterialId, byte skyLight, byte blockLight, @@ -122,20 +117,26 @@ public class ColumnBox // add top and bottom faces // //==========================// - boolean skipTop = RenderDataPointUtil.doesDataPointExist(topData) - && (RenderDataPointUtil.getYMin(topData) == maxY) - && !isTopTransparent; - if (!skipTop) + // top face { - builder.addQuadUp(minX, maxY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.UP)), irisBlockMaterialId, skyLightTop, blockLight); + boolean skipTop = RenderDataPointUtil.doesDataPointExist(topData) + && (RenderDataPointUtil.getYMin(topData) == maxY) + && !isTopTransparent; + if (!skipTop) + { + builder.addQuadUp(minX, maxY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.UP)), irisBlockMaterialId, skyLightTop, blockLight); + } } - boolean skipBottom = RenderDataPointUtil.doesDataPointExist(bottomData) - && (RenderDataPointUtil.getYMax(bottomData) == minY) - && !isBottomTransparent; - if (!skipBottom) + // bottom face { - builder.addQuadDown(minX, minY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.DOWN)), irisBlockMaterialId, skyLightBot, blockLight); + boolean skipBottom = RenderDataPointUtil.doesDataPointExist(bottomData) + && (RenderDataPointUtil.getYMax(bottomData) == minY) + && !isBottomTransparent; + if (!skipBottom) + { + builder.addQuadDown(minX, minY, minZ, width, width, ColorUtil.applyShade(color, MC.getShade(EDhDirection.DOWN)), irisBlockMaterialId, skyLightBot, blockLight); + } } @@ -146,84 +147,119 @@ public class ColumnBox // NORTH face { - ColumnArrayView adjCol = adjData[EDhDirection.NORTH.ordinal() - 2]; // TODO can we use something other than ordinal-2? - boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.NORTH.ordinal() - 2]; + ColumnArrayView adjCol = adjData[EDhDirection.NORTH.compassIndex]; + boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.NORTH.compassIndex]; // if the adjacent column is null that generally means the adjacent area hasn't been generated yet if (adjCol == null) { // Add an adjacent face if this is opaque face or transparent over the void. if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.NORTH, minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj( + EDhDirection.NORTH, + minX, minY, minZ, + width, yHeight, + color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.NORTH, minX, minY, minZ, width, yHeight, + makeAdjVerticalQuad( + builder, phantomArrayCheckout, + adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.NORTH, + minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } // SOUTH face { - ColumnArrayView adjCol = adjData[EDhDirection.SOUTH.ordinal() - 2]; - boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.SOUTH.ordinal() - 2]; + ColumnArrayView adjCol = adjData[EDhDirection.SOUTH.compassIndex]; + boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.SOUTH.compassIndex]; if (adjCol == null) { if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.SOUTH, minX, minY, maxZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj( + EDhDirection.SOUTH, + minX, minY, maxZ, + width, yHeight, + color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.SOUTH, minX, minY, maxZ, width, yHeight, + makeAdjVerticalQuad( + builder, phantomArrayCheckout, + adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.SOUTH, + minX, minY, maxZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } // WEST face { - ColumnArrayView adjCol = adjData[EDhDirection.WEST.ordinal() - 2]; - boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.WEST.ordinal() - 2]; + ColumnArrayView adjCol = adjData[EDhDirection.WEST.compassIndex]; + boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.WEST.compassIndex]; if (adjCol == null) { if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.WEST, minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj( + EDhDirection.WEST, + minX, minY, minZ, + width, yHeight, + color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.WEST, minX, minY, minZ, width, yHeight, + makeAdjVerticalQuad( + builder, phantomArrayCheckout, + adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.WEST, + minX, minY, minZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } // EAST face { - ColumnArrayView adjCol = adjData[EDhDirection.EAST.ordinal() - 2]; - boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.EAST.ordinal() - 2]; + ColumnArrayView adjCol = adjData[EDhDirection.EAST.compassIndex]; + boolean adjSameDetailLevel = isAdjDataSameDetailLevel[EDhDirection.EAST.compassIndex]; if (adjCol == null) { if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.EAST, maxX, minY, minZ, width, yHeight, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); + builder.addQuadAdj( + EDhDirection.EAST, + maxX, minY, minZ, + width, yHeight, + color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else { - makeAdjVerticalQuad(builder, adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.EAST, maxX, minY, minZ, width, yHeight, + makeAdjVerticalQuad( + builder, phantomArrayCheckout, + adjCol, adjSameDetailLevel, caveCullingMaxY, EDhDirection.EAST, + maxX, minY, minZ, width, yHeight, color, irisBlockMaterialId, blockLight); } } } private static void makeAdjVerticalQuad( - LodQuadBuilder builder, @NotNull ColumnArrayView adjColumnView, boolean adjacentIsSameDetailLevel, int caveCullingMaxY, EDhDirection direction, + LodQuadBuilder builder, PhantomArrayListCheckout phantomArrayCheckout, + @NotNull ColumnArrayView adjColumnView, boolean adjacentIsSameDetailLevel, int caveCullingMaxY, EDhDirection direction, short x, short yMin, short z, short horizontalWidth, short ySize, int color, byte irisBlockMaterialId, byte blockLight) { + // pooled arrays + LongArrayList segments = phantomArrayCheckout.getLongArray(0, 0); + LongArrayList newSegments = phantomArrayCheckout.getLongArray(1, 0); + + + //==================// // create face with // // no adjacent data // @@ -240,30 +276,29 @@ public class ColumnBox - //===========================// - // Build Y-range segments // - // with their sky light // - //===========================// + //=================================// + // determine face visibility/light // + //=================================// boolean transparencyEnabled = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled; boolean inputTransparent = ColorUtil.getAlpha(color) < 255 && transparencyEnabled; short yMax = (short) (yMin + ySize); - // List to store segments: [startY, endY, skyLight] - ArrayList segments = new ArrayList<>(); int adjCount = adjColumnView.size(); // Start with the entire range at max light - segments.add(new YSegment(yMin, yMax, LodUtil.MAX_MC_LIGHT)); + segments.add(YSegmentUtil.encode(yMin, yMax, LodUtil.MAX_MC_LIGHT)); - // Process each adjacent datapoint and split/update segments + // Process each adjacent datapoint and split segments as needed for (int adjIndex = 0; adjIndex < adjCount; adjIndex++) { long adjPoint = adjColumnView.get(adjIndex); short adjMinY = RenderDataPointUtil.getYMin(adjPoint); short adjMaxY = RenderDataPointUtil.getYMax(adjPoint); + // skip empty adjacent points + // or points below this one if (!RenderDataPointUtil.doesDataPointExist(adjPoint) || RenderDataPointUtil.hasZeroHeight(adjPoint) || yMax <= adjMinY) @@ -271,6 +306,7 @@ public class ColumnBox continue; } + long adjAbovePoint = (adjIndex != 0) ? adjColumnView.get(adjIndex - 1) : RenderDataPointUtil.EMPTY_DATA; long adjBelowPoint = (adjIndex + 1 < adjCount) ? adjColumnView.get(adjIndex + 1) : RenderDataPointUtil.EMPTY_DATA; @@ -306,13 +342,13 @@ public class ColumnBox // Apply light to the range [adjMinY, adjMaxY) - applyLightToRange(segments, adjMinY, adjMaxY, lightToApply); + applyLightToRange(segments, newSegments, adjMinY, adjMaxY, lightToApply); // Fill overhang area [adjMaxY, adjAboveMinY) with adjSkyLight - int adjAboveMinY = RenderDataPointUtil.getYMin(adjAbovePoint); + short adjAboveMinY = RenderDataPointUtil.getYMin(adjAbovePoint); if (adjMaxY < adjAboveMinY) { - applyLightToRange(segments, adjMaxY, adjAboveMinY, adjSkyLight); + applyLightToRange(segments, newSegments, adjMaxY, adjAboveMinY, adjSkyLight); } } @@ -323,27 +359,42 @@ public class ColumnBox // from segments // //=======================// - for (YSegment seg : segments) + for (int i = 0; i < segments.size(); i++) { + long segment = segments.getLong(i); tryAddVerticalFaceWithSkyLightToBuilder( builder, direction, x, z, horizontalWidth, color, irisBlockMaterialId, blockLight, - seg.skyLight, inputTransparent, seg.endY, seg.startY + YSegmentUtil.getSkyLight(segment), inputTransparent, YSegmentUtil.getEndY(segment), YSegmentUtil.getStartY(segment) ); } } - // Apply a light value to a Y range, splitting segments as needed - private static void applyLightToRange(ArrayList segments, int rangeStart, int rangeEnd, byte newLight) + /** + * Apply the new light value over the given y range, + * splitting segments as needed + *

+ * source: claude.ai + */ + private static void applyLightToRange( + LongArrayList segments, LongArrayList newSegments, + short rangeStart, short rangeEnd, + byte newLight) { - ArrayList newSegments = new ArrayList<>(); + // clear the pooled array that the new segments will go into + newSegments.clear(); - for (YSegment seg : segments) + for (int i = 0; i < segments.size(); i++) { + long seg = segments.getLong(i); + short endY = YSegmentUtil.getEndY(seg); + short startY = YSegmentUtil.getStartY(seg); + byte skyLight = YSegmentUtil.getSkyLight(seg); + // No overlap - if (seg.endY <= rangeStart - || seg.startY >= rangeEnd) + if (endY <= rangeStart + || startY >= rangeEnd) { newSegments.add(seg); continue; @@ -352,21 +403,21 @@ public class ColumnBox // Partial or complete overlap - need to split // Part before the range - if (seg.startY < rangeStart) + if (startY < rangeStart) { - newSegments.add(new YSegment(seg.startY, rangeStart, seg.skyLight)); + newSegments.add(YSegmentUtil.encode(startY, rangeStart, skyLight)); } // Overlapping part - take minimum light - int overlapStart = Math.max(seg.startY, rangeStart); - int overlapEnd = Math.min(seg.endY, rangeEnd); - byte minLight = (byte) Math.min(newLight, seg.skyLight); - newSegments.add(new YSegment(overlapStart, overlapEnd, minLight)); + short overlapStart = (short)Math.max(startY, rangeStart); + short overlapEnd = (short)Math.min(endY, rangeEnd); + byte minLight = (byte) Math.min(newLight, skyLight); + newSegments.add(YSegmentUtil.encode(overlapStart, overlapEnd, minLight)); // Part after the range - if (seg.endY > rangeEnd) + if (endY > rangeEnd) { - newSegments.add(new YSegment(rangeEnd, seg.endY, seg.skyLight)); + newSegments.add(YSegmentUtil.encode(rangeEnd, endY, skyLight)); } } @@ -412,23 +463,13 @@ public class ColumnBox - private static class YSegment - { - int startY; - int endY; - byte skyLight; - - YSegment(int startY, int endY, byte skyLight) - { - this.startY = startY; - this.endY = endY; - this.skyLight = skyLight; - } - } + //================// + // helper classes // + //================// - - /** - * @see com.seibel.distanthorizons.core.util.FullDataPointUtil + /** + * encodes height/light data into a long + * to reduce object allocations. */ private static class YSegmentUtil { @@ -456,11 +497,10 @@ public class ColumnBox public static short getStartY(long data) { return (short) ((data >> START_Y_OFFSET) & START_Y_MASK); } public static short getEndY(long data) { return (short) ((data >> END_Y_OFFSET) & END_Y_MASK); } - public static short getSkyLight(long data) { return (short) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } + public static byte getSkyLight(long data) { return (byte) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } } - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 7d8cd325f..41c3ace35 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -27,6 +27,8 @@ import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListCheckout; +import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.blockPos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.glObject.GLProxy; @@ -34,7 +36,6 @@ import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import java.util.concurrent.CompletableFuture; @@ -47,6 +48,8 @@ public class ColumnRenderBufferBuilder { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); + public static final PhantomArrayListPool ARRAY_LIST_POOL = new PhantomArrayListPool("Column Buffer Builder"); + //==============// @@ -104,182 +107,186 @@ public class ColumnRenderBufferBuilder // build each column // //===================// - byte thisDetailLevel = renderSource.getDataDetailLevel(); - for (int relX = 0; relX < ColumnRenderSource.WIDTH; relX++) + // pooled arrays for ColumnBox use + try (PhantomArrayListCheckout phantomArrayCheckout = ARRAY_LIST_POOL.checkoutArrays(0, 0, 2)) { - for (int relZ = 0; relZ < ColumnRenderSource.WIDTH; relZ++) + byte thisDetailLevel = renderSource.getDataDetailLevel(); + for (int relX = 0; relX < ColumnRenderSource.WIDTH; relX++) { - // ignore empty/null columns - ColumnArrayView columnRenderData = renderSource.getVerticalDataPointView(relX, relZ); - if (columnRenderData.size() == 0 - || !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0)) - || RenderDataPointUtil.hasZeroHeight(columnRenderData.get(0))) + for (int relZ = 0; relZ < ColumnRenderSource.WIDTH; relZ++) { - continue; - } - - - - //=============// - // debug limit // - //=============// - - // can be used to limit the buffer building to a specific relative position. - // useful for debugging a single column - if (columnBuilderDebugEnabled) - { - int wantedX = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugXRow.get(); - if (wantedX >= 0 && relX != wantedX) + // ignore empty/null columns + ColumnArrayView columnRenderData = renderSource.getVerticalDataPointView(relX, relZ); + if (columnRenderData.size() == 0 + || !RenderDataPointUtil.doesDataPointExist(columnRenderData.get(0)) + || RenderDataPointUtil.hasZeroHeight(columnRenderData.get(0))) { continue; } - int wantedZ = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugZRow.get(); - if (wantedZ >= 0 && relZ != wantedZ) + + + + //=============// + // debug limit // + //=============// + + // can be used to limit the buffer building to a specific relative position. + // useful for debugging a single column + if (columnBuilderDebugEnabled) { - continue; - } - } - - - - //==================================// - // get adjacent render data columns // - //==================================// - - ColumnArrayView[] adjColumnViews = new ColumnArrayView[EDhDirection.CARDINAL_COMPASS.length]; - for (EDhDirection lodDirection : EDhDirection.CARDINAL_COMPASS) - { - try - { - int xAdj = relX + lodDirection.normal.x; - int zAdj = relZ + lodDirection.normal.z; - boolean isCrossRenderSourceBoundary = - (xAdj < 0 || xAdj >= ColumnRenderSource.WIDTH) || - (zAdj < 0 || zAdj >= ColumnRenderSource.WIDTH); - - ColumnRenderSource adjRenderSource; - byte adjDetailLevel; - - - - //=========================// - // get the adjacent render // - // source if present // - //=========================// - - if (!isCrossRenderSourceBoundary) + int wantedX = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugXRow.get(); + if (wantedX >= 0 && relX != wantedX) { - // the adjacent position is inside this same render source - adjRenderSource = renderSource; - adjDetailLevel = thisDetailLevel; + continue; } - else - { - // the adjacent position is outside this render source - - // skip empty sections - adjRenderSource = adjRegions[lodDirection.ordinal() - 2]; - if (adjRenderSource == null) - { - continue; - } - - adjDetailLevel = adjRenderSource.getDataDetailLevel(); - if (adjDetailLevel == thisDetailLevel) - { - // if the adjacent position is outside this render source, - // wrap the position around so it's inside the adjacent source - - if (xAdj < 0) - { - xAdj += ColumnRenderSource.WIDTH; - } - if (xAdj >= ColumnRenderSource.WIDTH) - { - xAdj -= ColumnRenderSource.WIDTH; - } - - if (zAdj < 0) - { - zAdj += ColumnRenderSource.WIDTH; - } - if (zAdj >= ColumnRenderSource.WIDTH) - { - zAdj -= ColumnRenderSource.WIDTH; - } - } - } - - - - //========================// - // get the adjacent views // - //========================// - - // the old logic handled additional cases, but they never appeared to fire, - // so just these two cases should be fine - boolean expectedDetailLevels = (adjDetailLevel == thisDetailLevel) || (adjDetailLevel > thisDetailLevel); - if (!expectedDetailLevels) - { - LodUtil.assertNotReach("Mismatch between adjacent detail level ["+adjDetailLevel+"] and this render source's detail level ["+thisDetailLevel+"]. Detail levels should be adj >= this."); - } - - adjColumnViews[lodDirection.ordinal() - 2] = adjRenderSource.getVerticalDataPointView(xAdj, zAdj); - } - catch (RuntimeException e) - { - LOGGER.warn("Failed to get adj data for relative pos: [" + thisDetailLevel + ":" + relX + "," + relZ + "] at [" + lodDirection + "], Error: [" + e.getMessage() + "].", e); - } - } // for adjacent directions - - - - //==========================// - // build this render column // - //==========================// - - ColumnRenderSource.DebugSourceFlag debugSourceFlag = renderSource.debugGetFlag(relX, relZ); - - for (int i = 0; i < columnRenderData.size(); i++) - { - // can be uncommented to limit which vertical LOD is generated - if (Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugEnable.get()) - { - int wantedColumnIndex = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugColumnIndex.get(); - if (wantedColumnIndex >= 0 - && i != wantedColumnIndex) + int wantedZ = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugZRow.get(); + if (wantedZ >= 0 && relZ != wantedZ) { continue; } } - long data = columnRenderData.get(i); - // If the data is not render-able (Void or non-existing) we stop since there is - // no data left in this position - if (RenderDataPointUtil.hasZeroHeight(data) - || !RenderDataPointUtil.doesDataPointExist(data)) + + + //==================================// + // get adjacent render data columns // + //==================================// + + ColumnArrayView[] adjColumnViews = new ColumnArrayView[EDhDirection.CARDINAL_COMPASS.length]; + for (EDhDirection direction : EDhDirection.CARDINAL_COMPASS) { - break; + try + { + int xAdj = relX + direction.normal.x; + int zAdj = relZ + direction.normal.z; + boolean isCrossRenderSourceBoundary = + (xAdj < 0 || xAdj >= ColumnRenderSource.WIDTH) || + (zAdj < 0 || zAdj >= ColumnRenderSource.WIDTH); + + ColumnRenderSource adjRenderSource; + byte adjDetailLevel; + + + + //=========================// + // get the adjacent render // + // source if present // + //=========================// + + if (!isCrossRenderSourceBoundary) + { + // the adjacent position is inside this same render source + adjRenderSource = renderSource; + adjDetailLevel = thisDetailLevel; + } + else + { + // the adjacent position is outside this render source + + // skip empty sections + adjRenderSource = adjRegions[direction.compassIndex]; + if (adjRenderSource == null) + { + continue; + } + + adjDetailLevel = adjRenderSource.getDataDetailLevel(); + if (adjDetailLevel == thisDetailLevel) + { + // if the adjacent position is outside this render source, + // wrap the position around so it's inside the adjacent source + + if (xAdj < 0) + { + xAdj += ColumnRenderSource.WIDTH; + } + if (xAdj >= ColumnRenderSource.WIDTH) + { + xAdj -= ColumnRenderSource.WIDTH; + } + + if (zAdj < 0) + { + zAdj += ColumnRenderSource.WIDTH; + } + if (zAdj >= ColumnRenderSource.WIDTH) + { + zAdj -= ColumnRenderSource.WIDTH; + } + } + } + + + + //========================// + // get the adjacent views // + //========================// + + // the old logic handled additional cases, but they never appeared to fire, + // so just these two cases should be fine + boolean expectedDetailLevels = (adjDetailLevel == thisDetailLevel) || (adjDetailLevel > thisDetailLevel); + if (!expectedDetailLevels) + { + LodUtil.assertNotReach("Mismatch between adjacent detail level ["+adjDetailLevel+"] and this render source's detail level ["+thisDetailLevel+"]. Detail levels should be adj >= this."); + } + + adjColumnViews[direction.compassIndex] = adjRenderSource.getVerticalDataPointView(xAdj, zAdj); + } + catch (RuntimeException e) + { + LOGGER.warn("Failed to get adj data for relative pos: [" + thisDetailLevel + ":" + relX + "," + relZ + "] at [" + direction + "], Error: [" + e.getMessage() + "].", e); + } + } // for adjacent directions + + + + //==========================// + // build this render column // + //==========================// + + ColumnRenderSource.DebugSourceFlag debugSourceFlag = renderSource.debugGetFlag(relX, relZ); + + for (int i = 0; i < columnRenderData.size(); i++) + { + // can be uncommented to limit which vertical LOD is generated + if (Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugEnable.get()) + { + int wantedColumnIndex = Config.Client.Advanced.Debugging.ColumnBuilderDebugging.columnBuilderDebugColumnIndex.get(); + if (wantedColumnIndex >= 0 + && i != wantedColumnIndex) + { + continue; + } + } + + long data = columnRenderData.get(i); + // If the data is not render-able (Void or non-existing) we stop since there is + // no data left in this position + if (RenderDataPointUtil.hasZeroHeight(data) + || !RenderDataPointUtil.doesDataPointExist(data)) + { + break; + } + + long topDataPoint = (i - 1) >= 0 ? columnRenderData.get(i - 1) : RenderDataPointUtil.EMPTY_DATA; + long bottomDataPoint = (i + 1) < columnRenderData.size() ? columnRenderData.get(i + 1) : RenderDataPointUtil.EMPTY_DATA; + + addRenderDataPointToBuilder( + clientLevel, phantomArrayCheckout, + data, topDataPoint, bottomDataPoint, + adjColumnViews, isSameDetailLevel, + thisDetailLevel, relX, relZ, + quadBuilder, debugSourceFlag); } - long topDataPoint = (i - 1) >= 0 ? columnRenderData.get(i - 1) : RenderDataPointUtil.EMPTY_DATA; - long bottomDataPoint = (i + 1) < columnRenderData.size() ? columnRenderData.get(i + 1) : RenderDataPointUtil.EMPTY_DATA; - - addRenderDataPointToBuilder( - clientLevel, - data, topDataPoint, bottomDataPoint, - adjColumnViews, isSameDetailLevel, - thisDetailLevel, relX, relZ, - quadBuilder, debugSourceFlag); - } - - }// for z - }// for x + }// for z + }// for x + }// phantom checkout quadBuilder.mergeQuads(); } private static void addRenderDataPointToBuilder( - IDhClientLevel clientLevel, + IDhClientLevel clientLevel, PhantomArrayListCheckout phantomArrayCheckout, long renderData, long topRenderData, long bottomRenderData, ColumnArrayView[] adjColumnViews, boolean[] isSameDetailLevel, byte detailLevel, int renderSourceOffsetPosX, int renderSourceOffsetPosZ, @@ -411,7 +418,7 @@ public class ColumnRenderBufferBuilder } ColumnBox.addBoxQuadsToBuilder( - quadBuilder, clientLevel, + quadBuilder, phantomArrayCheckout, clientLevel, blockWidth, blockMaxY, blockMinX, blockMinY, blockMinZ, color, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index cb4100455..ef8813c05 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -175,9 +175,7 @@ public class FullDataToRenderDataTransformer ColumnArrayView newColumnArrayView = new ColumnArrayView(dataArrayList, fullDataLength, 0, fullDataLength); setRenderColumnView(levelWrapper, fullDataSource, blockX, blockZ, newColumnArrayView, fullDataColumn); - PerfRecorder.Timer vertSize = LodQuadTree.TRANSFORM_PERF_RECORDER.start("vertSize"); columnArrayView.changeVerticalSizeFrom(newColumnArrayView); - vertSize.end(); } finally { @@ -190,8 +188,6 @@ public class FullDataToRenderDataTransformer int blockX, int blockZ, ColumnArrayView renderColumnData, LongArrayList fullColumnData) { - PerfRecorder.Timer prep = LodQuadTree.TRANSFORM_PERF_RECORDER.start("prep"); - //===============// // config values // //===============// @@ -223,8 +219,6 @@ public class FullDataToRenderDataTransformer int blocklightToApplyToNextBlock = -1; int renderDataIndex = 0; - prep.end(); - //==================================// @@ -238,8 +232,6 @@ public class FullDataToRenderDataTransformer // goes from the top down for (int fullDataIndex = 0; fullDataIndex < fullColumnData.size(); fullDataIndex++) { - PerfRecorder.Timer fullParse = LodQuadTree.TRANSFORM_PERF_RECORDER.start("fullParse"); - long fullData = fullColumnData.getLong(fullDataIndex); int bottomY = FullDataPointUtil.getBottomY(fullData); @@ -275,8 +267,6 @@ public class FullDataToRenderDataTransformer continue; } - fullParse.end(); - //====================// @@ -284,57 +274,48 @@ public class FullDataToRenderDataTransformer // cave culling check // //====================// - PerfRecorder.Timer caveCull = LodQuadTree.TRANSFORM_PERF_RECORDER.start("caveCull"); - - try + boolean ignoreBlock = blockStatesToIgnore.contains(block); + boolean caveBlock = caveBlockStatesToIgnore.contains(block); // TODO caves should also ignore transparent/non-solid blocks (IE grass and plants) wthout each being defined + if (caveBlock) { - boolean ignoreBlock = blockStatesToIgnore.contains(block); - boolean caveBlock = caveBlockStatesToIgnore.contains(block); // TODO caves should also ignore transparent/non-solid blocks (IE grass and plants) wthout each being defined - if (caveBlock) + if (caveCullingEnabled + // assume this data point is underground if it has no sky-light + && skyLight == LodUtil.MIN_MC_LIGHT + // ignore caves above a certain height to prevent floating islands from having walls underneath them + && topY < caveCullingMaxY + // cave culling shouldn't happen when at the top of the world + && renderDataIndex != 0 && fullDataIndex != 0 + // cave culling can't happen when at the bottom of the world + && (fullDataIndex + 1) < fullColumnData.size()) { - if (caveCullingEnabled - // assume this data point is underground if it has no sky-light - && skyLight == LodUtil.MIN_MC_LIGHT - // ignore caves above a certain height to prevent floating islands from having walls underneath them - && topY < caveCullingMaxY - // cave culling shouldn't happen when at the top of the world - && renderDataIndex != 0 && fullDataIndex != 0 - // cave culling can't happen when at the bottom of the world - && (fullDataIndex + 1) < fullColumnData.size()) + // we need to get the next sky/block lights because + // the air block here will always have a light of 0/0 due to only the top of the LOD's light being saved. + long nextFullData = fullColumnData.getLong(fullDataIndex + 1); + int nextSkyLight = FullDataPointUtil.getSkyLight(nextFullData); + + if (nextSkyLight == LodUtil.MIN_MC_LIGHT + && ColorUtil.getAlpha(lastColor) == 255) { - // we need to get the next sky/block lights because - // the air block here will always have a light of 0/0 due to only the top of the LOD's light being saved. - long nextFullData = fullColumnData.getLong(fullDataIndex + 1); - int nextSkyLight = FullDataPointUtil.getSkyLight(nextFullData); - - if (nextSkyLight == LodUtil.MIN_MC_LIGHT - && ColorUtil.getAlpha(lastColor) == 255) - { - // replace the previous block with new bottom - long columnData = renderColumnData.get(renderDataIndex - 1); - columnData = RenderDataPointUtil.setYMin(columnData, bottomY); - renderColumnData.set(renderDataIndex - 1, columnData); - } - - continue; + // replace the previous block with new bottom + long columnData = renderColumnData.get(renderDataIndex - 1); + columnData = RenderDataPointUtil.setYMin(columnData, bottomY); + renderColumnData.set(renderDataIndex - 1, columnData); } - - if (ignoreBlock) - { - // this is a merged block and a cave block, so it should never be rendered - continue; - } + continue; } - else if (ignoreBlock) + + + if (ignoreBlock) { - // this is an ignored block, but shouldn't be merged like a cave block + // this is a merged block and a cave block, so it should never be rendered continue; } } - finally + else if (ignoreBlock) { - caveCull.end(); + // this is an ignored block, but shouldn't be merged like a cave block + continue; } @@ -350,9 +331,7 @@ public class FullDataToRenderDataTransformer { if (colorBelowWithAvoidedBlocks) { - PerfRecorder.Timer nonSolid = LodQuadTree.TRANSFORM_PERF_RECORDER.start("color-NonSolid"); int tempColor = levelWrapper.getBlockColor(mutableBlockPos, biome, fullDataSource, block); - nonSolid.end(); // don't transfer the color when alpha is 0 // this prevents issues if grass is transparent @@ -372,10 +351,8 @@ public class FullDataToRenderDataTransformer int color; if (colorToApplyToNextBlock == -1) { - PerfRecorder.Timer colorTimer = LodQuadTree.TRANSFORM_PERF_RECORDER.start("color"); // use this block's color color = levelWrapper.getBlockColor(mutableBlockPos, biome, fullDataSource, block); - colorTimer.end(); } else { @@ -391,7 +368,6 @@ public class FullDataToRenderDataTransformer //=============================// // merge same-colored adjacent // //=============================// - PerfRecorder.Timer mergeSame = LodQuadTree.TRANSFORM_PERF_RECORDER.start("mergeSame"); // check if they share a top-bottom face and if they have same color if (color == lastColor @@ -413,8 +389,6 @@ public class FullDataToRenderDataTransformer } lastBottom = bottomY; lastColor = color; - - mergeSame.end(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 3fed7c1a5..b28a14105 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -97,8 +97,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen // TODO should be removed once James is done testing @Deprecated public static final PerfRecorder FILE_PERF_RECORDER = new PerfRecorder("File"); - @Deprecated - public static final PerfRecorder TRANSFORM_PERF_RECORDER = new PerfRecorder("Transform"); /** the smallest numerical detail level number that can be rendered */ private byte maxLeafRenderDetailLevel; @@ -133,8 +131,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen this.beaconRenderHandler = (genericObjectRenderer != null) ? new BeaconRenderHandler(genericObjectRenderer) : null; FILE_PERF_RECORDER.clear(); - TRANSFORM_PERF_RECORDER.clear(); - COL_BOX_PERF_RECORDER.clear(); } @@ -160,8 +156,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen FILE_PERF_RECORDER.tryLog(); - TRANSFORM_PERF_RECORDER.tryLog(); - COL_BOX_PERF_RECORDER.tryLog(); // this shouldn't be updated while the tree is being iterated through diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 4984bdb7b..284c8ded7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -287,16 +287,16 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable ColumnRenderSource westRenderSource = adjacentLoadFutures[3].get()) { ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.CARDINAL_COMPASS.length]; - adjacentRenderSections[EDhDirection.NORTH.ordinal() - 2] = northRenderSource; - adjacentRenderSections[EDhDirection.SOUTH.ordinal() - 2] = southRenderSource; - adjacentRenderSections[EDhDirection.EAST.ordinal() - 2] = eastRenderSource; - adjacentRenderSections[EDhDirection.WEST.ordinal() - 2] = westRenderSource; + adjacentRenderSections[EDhDirection.NORTH.compassIndex] = northRenderSource; + adjacentRenderSections[EDhDirection.SOUTH.compassIndex] = southRenderSource; + adjacentRenderSections[EDhDirection.EAST.compassIndex] = eastRenderSource; + adjacentRenderSections[EDhDirection.WEST.compassIndex] = westRenderSource; boolean[] adjIsSameDetailLevel = new boolean[EDhDirection.CARDINAL_COMPASS.length]; - adjIsSameDetailLevel[EDhDirection.NORTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.NORTH); - adjIsSameDetailLevel[EDhDirection.SOUTH.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.SOUTH); - adjIsSameDetailLevel[EDhDirection.EAST.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.EAST); - adjIsSameDetailLevel[EDhDirection.WEST.ordinal() - 2] = this.isAdjacentPosSameDetailLevel(EDhDirection.WEST); + adjIsSameDetailLevel[EDhDirection.NORTH.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.NORTH); + adjIsSameDetailLevel[EDhDirection.SOUTH.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.SOUTH); + adjIsSameDetailLevel[EDhDirection.EAST.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.EAST); + adjIsSameDetailLevel[EDhDirection.WEST.compassIndex] = this.isAdjacentPosSameDetailLevel(EDhDirection.WEST); // the render sources are only needed by this synchronous method, // then they can be closed From 62359e3dde36e6241a7498f68e6b12a7979b2fa4 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 19:08:30 -0600 Subject: [PATCH 18/24] remove LOD load pref logging --- .../core/render/LodQuadTree.java | 8 -------- .../core/render/LodRenderSection.java | 18 ------------------ 2 files changed, 26 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index b28a14105..80d3b1a95 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -94,10 +94,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen @Nullable public final BeaconRenderHandler beaconRenderHandler; - // TODO should be removed once James is done testing - @Deprecated - public static final PerfRecorder FILE_PERF_RECORDER = new PerfRecorder("File"); - /** the smallest numerical detail level number that can be rendered */ private byte maxLeafRenderDetailLevel; /** the largest numerical detail level number that can be rendered */ @@ -130,8 +126,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen GenericObjectRenderer genericObjectRenderer = this.level.getGenericRenderer(); this.beaconRenderHandler = (genericObjectRenderer != null) ? new BeaconRenderHandler(genericObjectRenderer) : null; - FILE_PERF_RECORDER.clear(); - } @@ -155,8 +149,6 @@ public class LodQuadTree extends QuadTree implements IDebugRen } - FILE_PERF_RECORDER.tryLog(); - // this shouldn't be updated while the tree is being iterated through this.updateDetailLevelVariables(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 284c8ded7..527b7b9ec 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -128,9 +128,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable private boolean checkedIfFullDataSourceExists = false; private boolean fullDataSourceExists = false; - @Deprecated - public final PerfRecorder filePerfRecorder = LodQuadTree.FILE_PERF_RECORDER; - //=============// @@ -254,8 +251,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable LodQuadBuilder lodQuadBuilder = new LodQuadBuilder(enableTransparency, this.level.getClientLevelWrapper()); - PerfRecorder.Timer getAdj = this.filePerfRecorder.start("getAdj"); - // get the adjacent positions // needs to be done async to prevent threads waiting on the same positions to be processed final CompletableFuture[] adjacentLoadFutures = new CompletableFuture[4]; @@ -279,8 +274,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable return CompletableFuture.allOf(adjacentLoadFutures).thenRun(() -> { - getAdj.end(); - try (ColumnRenderSource northRenderSource = adjacentLoadFutures[0].get(); ColumnRenderSource southRenderSource = adjacentLoadFutures[1].get(); ColumnRenderSource eastRenderSource = adjacentLoadFutures[2].get(); @@ -300,13 +293,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // the render sources are only needed by this synchronous method, // then they can be closed - PerfRecorder.Timer makeRender = this.filePerfRecorder.start("makeRender"); ColumnRenderBufferBuilder.makeLodRenderData(lodQuadBuilder, thisRenderSource, this.level, adjacentRenderSections, adjIsSameDetailLevel); - makeRender.end(); - - PerfRecorder.Timer upload = this.filePerfRecorder.start("upload"); this.uploadToGpuAsync(lodQuadBuilder); - upload.end(); } catch (Exception e) { @@ -350,8 +338,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable CompletableFuture loadFuture = new CompletableFuture<>(); executor.execute(() -> { - PerfRecorder.Timer getFull = this.filePerfRecorder.start("getFull"); - // generate new render source try (FullDataSourceV2 fullDataSource = // no direction means get the center LOD @@ -359,12 +345,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable ? this.fullDataSourceProvider.get(finalPos) : this.fullDataSourceProvider.getAdjForDirection(finalPos, direction.opposite())) { - getFull.end(); - - PerfRecorder.Timer transform = this.filePerfRecorder.start("transform"); ColumnRenderSource columnRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.levelWrapper); loadFuture.complete(columnRenderSource); - transform.end(); } catch (Exception e) { From 97442f8833711a98c0f9bd5ce30c0ea36ca99823 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 8 Nov 2025 19:11:56 -0600 Subject: [PATCH 19/24] Fix config min/max validation default setup --- .../com/seibel/distanthorizons/core/config/ConfigHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java index 32f0e58fb..2fd64dabd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/ConfigHandler.java @@ -98,7 +98,7 @@ public class ConfigHandler * Disables the minimum and maximum validation.
* Fun to use, but should be disabled by default. */ - public boolean runMinMaxValidation = Config.Client.Advanced.Debugging.allowUnsafeValues.get(); + public boolean runMinMaxValidation = true; @@ -123,6 +123,7 @@ public class ConfigHandler this.initNestedClass(Config.class, ""); // Init root category this.configFileHandler.loadFromFile(); + this.runMinMaxValidation = !Config.Client.Advanced.Debugging.allowUnsafeValues.get(); this.isLoaded = true; LOGGER.info("[" + ModInfo.NAME + "] Config initialised"); From 767753c00421ca5883cf67c81d28a39b45891e5e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 10 Nov 2025 06:56:24 -0600 Subject: [PATCH 20/24] add logging to infinite repo unit test --- .../java/tests/DhFullDataSourceRepoTests.java | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/core/src/test/java/tests/DhFullDataSourceRepoTests.java b/core/src/test/java/tests/DhFullDataSourceRepoTests.java index 8a03dd587..055933372 100644 --- a/core/src/test/java/tests/DhFullDataSourceRepoTests.java +++ b/core/src/test/java/tests/DhFullDataSourceRepoTests.java @@ -30,6 +30,7 @@ import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.objects.DataCorruptedException; import it.unimi.dsi.fastutil.bytes.ByteArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.junit.Assert; @@ -42,6 +43,8 @@ import java.util.Arrays; import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; /** * Can also be used to test if there are memory leaks in SQLite @@ -239,6 +242,9 @@ public class DhFullDataSourceRepoTests { System.out.println("Initial save/get success, starting long update test for GC validation..."); + AtomicLong lastLogMsTime = new AtomicLong(0); + LongAdder iterateCount = new LongAdder(); + int poolSize = Runtime.getRuntime().availableProcessors(); CompletableFuture[] futures = new CompletableFuture[poolSize]; ThreadPoolExecutor pool = ThreadUtil.makeThreadPool(poolSize, "test pool"); @@ -260,21 +266,47 @@ public class DhFullDataSourceRepoTests } // new position so each DTO is different and saved to a different row in the DB - long threadPos = DhSectionPos.encode((byte)0, finalThreadIndex, 0); + long threadPos = DhSectionPos.encode((byte)6, finalThreadIndex, 0); threadDto.pos = threadPos; + repo.save(threadDto); // runs significantly faster if we don't save + Assert.assertNotNull(threadDto); + // run for a long time - for (int j = 0; j < 1_000_000; j++) + for (int j = 0; j < 100_000_000; j++) { - repo.save(threadDto); // runs significantly faster if we don't save - try (FullDataSourceV2DTO pooledDto = repo.getByKey(threadPos)) { - Assert.assertNotNull(threadDto); Assert.assertEquals(pooledDto.pos, threadDto.pos); - assertArraysAreEqual(pooledDto.compressedDataByteArray, threadDto.compressedDataByteArray); - assertArraysAreEqual(pooledDto.compressedColumnGenStepByteArray, threadDto.compressedColumnGenStepByteArray); - assertArraysAreEqual(pooledDto.compressedWorldCompressionModeByteArray, threadDto.compressedWorldCompressionModeByteArray); + Assert.assertFalse(pooledDto.compressedDataByteArray.isEmpty()); + Assert.assertFalse(pooledDto.compressedColumnGenStepByteArray.isEmpty()); + Assert.assertFalse(pooledDto.compressedWorldCompressionModeByteArray.isEmpty()); + + try (FullDataSourceV2 dataSource = pooledDto.createUnitTestDataSource(); + FullDataSourceV2DTO compressedDto = FullDataSourceV2DTO.CreateFromDataSource(dataSource, EDhApiDataCompressionMode.Z_STD)) + { + repo.save(compressedDto); + + + + iterateCount.increment(); + + long time = System.currentTimeMillis(); + if (time - lastLogMsTime.get() > 30_000) + { + lastLogMsTime.set(time); + + Runtime runtime = Runtime.getRuntime(); + long free = runtime.freeMemory(); + long total = runtime.totalMemory(); + long max = runtime.maxMemory(); + + System.out.println("count: "+iterateCount.sum()+"\tfree: "+free+"\ttotal: "+total+"\tmax: "+max); + } + } + catch (Exception ignore) + { + } } } }, pool); From 6eb24ecde13ecd14ebb933faa4dd1907fe4c7801 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 10 Nov 2025 07:33:03 -0600 Subject: [PATCH 21/24] re-add GPU upload config including "none" --- .../api/enums/config/EDhApiGpuUploadMethod.java | 4 ++++ .../seibel/distanthorizons/core/config/Config.java | 7 +++++++ .../bufferBuilding/ColumnRenderBufferBuilder.java | 2 +- .../render/bufferBuilding/LodBufferContainer.java | 14 +++++++++----- .../core/render/glObject/GLProxy.java | 11 ++++++++++- .../core/render/glObject/buffer/GLBuffer.java | 7 ++++--- .../assets/distanthorizons/lang/en_us.json | 6 ++++++ 7 files changed, 41 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java index a2fca195b..ccd511013 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java @@ -44,6 +44,10 @@ public enum EDhApiGpuUploadMethod /** Fast rendering but may stutter when uploading. */ SUB_DATA(false, false), + /** Don't upload, only should be used for debugging */ + @Deprecated // TODO remove before release + NONE(false, false), + /** * May end up storing buffers in System memory.
* Fast rending if in GPU memory, slow if in system memory,
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 1cb36fae5..3ee1cc497 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -1083,6 +1083,13 @@ public class Config + "") .build(); + public static ConfigEntry glUploadMode = new ConfigEntry.Builder() + .set(EDhApiGpuUploadMethod.AUTO) + .comment("" + + "\n" + + "") + .build(); + } public static class ColumnBuilderDebugging diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 41c3ace35..a797f18be 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -65,7 +65,7 @@ public class ColumnRenderBufferBuilder { DhBlockPos minBlockPos = new DhBlockPos(DhSectionPos.getMinCornerBlockX(pos), clientLevel.getLevelWrapper().getMinHeight(), DhSectionPos.getMinCornerBlockZ(pos)); LodBufferContainer bufferContainer = new LodBufferContainer(pos, minBlockPos); - CompletableFuture uploadFuture = bufferContainer.makeAndUploadBuffersAsync(quadBuilder, GLProxy.getInstance().getGpuUploadMethod()); + CompletableFuture uploadFuture = bufferContainer.makeAndUploadBuffersAsync(quadBuilder); uploadFuture.whenComplete((uploadedBuffer, exception) -> { // clean up if not uploaded diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodBufferContainer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodBufferContainer.java index 4b5c92c89..7b03ccb09 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodBufferContainer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodBufferContainer.java @@ -82,7 +82,7 @@ public class LodBufferContainer implements AutoCloseable //==================// /** Should be run on a DH thread. */ - public synchronized CompletableFuture makeAndUploadBuffersAsync(LodQuadBuilder builder, EDhApiGpuUploadMethod gpuUploadMethod) + public synchronized CompletableFuture makeAndUploadBuffersAsync(LodQuadBuilder builder) { // separate variable to prevent race condition when checking null CompletableFuture future = this.uploadFuture; @@ -117,6 +117,8 @@ public class LodBufferContainer implements AutoCloseable throw new InterruptedException(); } + EDhApiGpuUploadMethod gpuUploadMethod = GLProxy.getInstance().getGpuUploadMethod(); + // upload on the render thread uploadBuffersDirect(this.vbos, opaqueBuffers, gpuUploadMethod); uploadBuffersDirect(this.vbosTransparent, transparentBuffers, gpuUploadMethod); @@ -177,7 +179,9 @@ public class LodBufferContainer implements AutoCloseable } return newVbos; } - private static void uploadBuffersDirect(GLVertexBuffer[] vbos, ArrayList byteBuffers, EDhApiGpuUploadMethod method) throws InterruptedException + private static void uploadBuffersDirect( + GLVertexBuffer[] vbos, ArrayList byteBuffers, + EDhApiGpuUploadMethod uploadMethod) throws InterruptedException { int vboIndex = 0; for (int i = 0; i < byteBuffers.size(); i++) @@ -191,7 +195,7 @@ public class LodBufferContainer implements AutoCloseable // get or create the VBO if (vbos[vboIndex] == null) { - vbos[vboIndex] = new GLVertexBuffer(method.useBufferStorage); + vbos[vboIndex] = new GLVertexBuffer(uploadMethod.useBufferStorage); } GLVertexBuffer vbo = vbos[vboIndex]; @@ -202,13 +206,13 @@ public class LodBufferContainer implements AutoCloseable try { vbo.bind(); - vbo.uploadBuffer(buffer, size / LodUtil.LOD_VERTEX_FORMAT.getByteSize(), method, FULL_SIZED_BUFFER); + vbo.uploadBuffer(buffer, size / LodUtil.LOD_VERTEX_FORMAT.getByteSize(), uploadMethod, FULL_SIZED_BUFFER); } catch (Exception e) { vbos[vboIndex] = null; vbo.close(); - LOGGER.error("Failed to upload buffer: ", e); + LOGGER.error("Failed to upload buffer. Error: ["+e.getMessage()+"].", e); } vboIndex++; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java index 2286a3429..dc4e5c6f1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java @@ -220,7 +220,16 @@ public class GLProxy return instance; } - public EDhApiGpuUploadMethod getGpuUploadMethod() { return this.preferredUploadMethod; } + public EDhApiGpuUploadMethod getGpuUploadMethod() + { + EDhApiGpuUploadMethod uploadOverride = Config.Client.Advanced.Debugging.OpenGl.glUploadMode.get(); + if (uploadOverride == EDhApiGpuUploadMethod.AUTO) + { + return this.preferredUploadMethod; + } + + return uploadOverride; + } public boolean runningOnRenderThread() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java index d45d61bae..4c16652b1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java @@ -27,7 +27,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; -import com.seibel.distanthorizons.core.util.math.UnitBytes; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftGLWrapper; import org.lwjgl.opengl.GL32; import org.lwjgl.opengl.GL44; @@ -187,7 +186,6 @@ public class GLBuffer implements AutoCloseable { LodUtil.assertNotReach("maxExpansionSize is [" + maxExpansionSize + "] but buffer size is [" + bbSize + "]!"); } - GLProxy.LOGGER.debug("Uploading buffer with ["+new UnitBytes(bbSize)+"]."); // Don't upload an empty buffer if (bbSize == 0) @@ -200,6 +198,8 @@ public class GLBuffer implements AutoCloseable switch (uploadMethod) { + case NONE: + return; case AUTO: LodUtil.assertNotReach("GpuUploadMethod AUTO must be resolved before call to uploadBuffer()!"); case BUFFER_STORAGE: @@ -379,6 +379,7 @@ public class GLBuffer implements AutoCloseable { int id = PHANTOM_TO_BUFFER_ID.get(phantomRef); destroyBufferIdAsync(id); + LOGGER.warn("Buffer Phantom collected, ID: ["+id+"]"); } phantomRef = PHANTOM_REFERENCE_QUEUE.poll(); @@ -386,7 +387,7 @@ public class GLBuffer implements AutoCloseable } catch (Exception e) { - LOGGER.error("Unexpected error in cleanup thread: [" + e.getMessage() + "].", e); + LOGGER.error("Unexpected error in buffer cleanup thread: [" + e.getMessage() + "].", e); } } } diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 6c7f85efa..813af98ee 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -509,6 +509,10 @@ "Validate Buffer IDs Before Rendering", "distanthorizons.config.client.advanced.debugging.openGl.validateBufferIdsBeforeRendering.@tooltip": "Massively reduces FPS. \nShould only be used if mysterious EXCEPTION_ACCESS_VIOLATION crashes are happening in DH's rendering code and you're attempting to troubleshoot it.", + "distanthorizons.config.client.advanced.debugging.openGl.glUploadMode": + "Uploade Mode", + "distanthorizons.config.client.advanced.debugging.openGl.glUploadMode.@tooltip": + "Only for debugging", @@ -1046,6 +1050,8 @@ "distanthorizons.config.enum.EDhApiGpuUploadMethod.AUTO": "Auto", + "distanthorizons.config.enum.EDhApiGpuUploadMethod.NONE": + "None", "distanthorizons.config.enum.EDhApiGpuUploadMethod.BUFFER_STORAGE": "Buffer storage", "distanthorizons.config.enum.EDhApiGpuUploadMethod.SUB_DATA": From b9746381eb11d39197d6a7fc813abdad1e3ea108 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 12 Nov 2025 07:21:54 -0600 Subject: [PATCH 22/24] Add varint encoding for full data Closes Merge !93 Thanks Ryan Hitchman! --- .../core/sql/dto/FullDataSourceV2DTO.java | 221 +++++++++++++++++- .../core/sql/dto/util/VarintUtil.java | 69 ++++++ .../java/tests/DhFullDataSourceRepoTests.java | 42 +++- core/src/test/java/tests/VarintTest.java | 94 ++++++++ 4 files changed, 416 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/VarintUtil.java create mode 100644 core/src/test/java/tests/VarintTest.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 2d7917a1b..d0446454f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -31,6 +31,7 @@ import com.seibel.distanthorizons.core.pooling.PhantomArrayListPool; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.network.INetworkObject; import com.seibel.distanthorizons.core.sql.dto.util.FullDataMinMaxPosUtil; +import com.seibel.distanthorizons.core.sql.dto.util.VarintUtil; import com.seibel.distanthorizons.core.util.BoolUtil; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.ListUtil; @@ -109,7 +110,7 @@ public class FullDataSourceV2DTO FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding(); // populate arrays - writeDataSourceDataArrayToBlob(dataSource.dataPoints, dto.compressedDataByteArray, compressionModeEnum); + writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedDataByteArray, compressionModeEnum); writeGenerationStepsToBlob(dataSource.columnGenerationSteps, dto.compressedColumnGenStepByteArray, compressionModeEnum); writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, dto.compressedWorldCompressionModeByteArray, compressionModeEnum); writeDataMappingToBlob(dataSource.mapping, dto.compressedMappingByteArray, compressionModeEnum); @@ -199,8 +200,8 @@ public class FullDataSourceV2DTO { // format validation // - if (DATA_FORMAT.V1_NO_ADJACENT_DATA != this.dataFormatVersion - && DATA_FORMAT.V2_LATEST != this.dataFormatVersion) + if (this.dataFormatVersion != DATA_FORMAT.V1_NO_ADJACENT_DATA + && this.dataFormatVersion != DATA_FORMAT.V2_LATEST) { throw new IllegalStateException("Data source population only supports formats: ["+DATA_FORMAT.V1_NO_ADJACENT_DATA +","+DATA_FORMAT.V2_LATEST +"], data format found: ["+this.dataFormatVersion+"]."); } @@ -234,7 +235,15 @@ public class FullDataSourceV2DTO { readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); readBlobToWorldCompressionMode(this.compressedWorldCompressionModeByteArray, dataSource.columnWorldCompressionMode, compressionModeEnum); - readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + + if (this.dataFormatVersion == 1) + { + readBlobToDataSourceDataArrayV1(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + } + else + { + readBlobToDataSourceDataArrayV2(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + } } else { @@ -398,7 +407,7 @@ public class FullDataSourceV2DTO } - private static void writeDataSourceDataArrayToBlob( + public static void writeDataSourceDataArrayToBlobV1( LongArrayList[] inputDataArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { @@ -435,7 +444,7 @@ public class FullDataSourceV2DTO outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); } } - private static void readBlobToDataSourceDataArray( + private static void readBlobToDataSourceDataArrayV1( ByteArrayList inputCompressedDataByteArray, LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { @@ -470,6 +479,206 @@ public class FullDataSourceV2DTO } } + private static void writeDataSourceDataArrayToBlobV2( + LongArrayList[] inputDataArray, ByteArrayList outputByteArray, + EDhApiDataCompressionMode compressionModeEnum) throws IOException + { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum)) + { + int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; + + // this method would be simpler if we allocated a bunch of temporary arrays, + // but we're trying to avoid garbage. + + // 1. column lengths + for (int xz = 0; xz < dataArrayLength; xz++) + { + LongArrayList col = inputDataArray[xz]; + int size = col != null ? col.size() : 0; + VarintUtil.writeVarint(compressedOut, size); + } + + // 2. column ids, with "is lit" and "is discontinuous" bits + int previousBottomY = 0; + + for (int xz = 0; xz < dataArrayLength; xz++) + { + LongArrayList col = inputDataArray[xz]; + int size = col != null ? col.size() : 0; + for (int y = 0; y < size; y++) + { + long data = col.getLong(y); + + int id = FullDataPointUtil.getId(data); + int height = FullDataPointUtil.getHeight(data); + int bottomY = FullDataPointUtil.getBottomY(data); + + boolean hasLight = (FullDataPointUtil.getBlockLight(data) | FullDataPointUtil.getSkyLight(data)) != LodUtil.MIN_MC_LIGHT; + + // all datapoints are contiguous, with no gaps + // so having both height and bottomY is redundant. We could store the prediction + // in an array, but it's much cheaper to just recompute it later. + int expectedBottomY = previousBottomY - height; + boolean hasDiscontinuity = bottomY != expectedBottomY; + previousBottomY = bottomY; + + VarintUtil.writeVarint(compressedOut, (id << 2) | (hasLight ? 2 : 0) | (hasDiscontinuity ? 1 : 0)); + } + } + + // 3. heights + for (int xz = 0; xz < dataArrayLength; xz++) + { + LongArrayList col = inputDataArray[xz]; + int size = (col != null) ? col.size() : 0; + for (int y = 0; y < size; y++) + { + long data = col.getLong(y); + VarintUtil.writeVarint(compressedOut, FullDataPointUtil.getHeight(data)); + } + } + + // 4. bottomY (only the mis-predicted ones) + previousBottomY = 0; + for (int xz = 0; xz < dataArrayLength; xz++) + { + LongArrayList col = inputDataArray[xz]; + int size = (col != null) ? col.size() : 0; + for (int y = 0; y < size; y++) + { + long data = col.getLong(y); + + int height = FullDataPointUtil.getHeight(data); + int bottomY = FullDataPointUtil.getBottomY(data); + + int expectedBottomY = previousBottomY - height; + if (bottomY != expectedBottomY) + { + VarintUtil.writeVarint(compressedOut, VarintUtil.zigzagEncode(bottomY - expectedBottomY)); + } + previousBottomY = bottomY; + } + } + + // 5. packed Light (only lit sections) + for (int xz = 0; xz < dataArrayLength; xz++) + { + LongArrayList col = inputDataArray[xz]; + int size = col != null ? col.size() : 0; + for (int y = 0; y < size; y++) + { + long data = col.getLong(y); + int blockLight = FullDataPointUtil.getBlockLight(data); + int skyLight = FullDataPointUtil.getSkyLight(data); + byte packedLight = (byte) ((blockLight << 4) | skyLight); + if (packedLight != 0) + { + compressedOut.writeByte(packedLight); + } + } + } + + compressedOut.flush(); + byteArrayOutputStream.close(); + outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); + } + } + private static void readBlobToDataSourceDataArrayV2( + ByteArrayList inputCompressedDataByteArray, + LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) + throws IOException, DataCorruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements()); + try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum)) + { + // 1. column counts, preallocate + int numColumns = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; + for (int i = 0; i < numColumns; i++) + { + int count = VarintUtil.readVarint(compressedIn); + ListUtil.clearAndSetSize(outputDataLongArray[i], count); + } + + // 2. ids and flags for min_y and light + for (LongArrayList col : outputDataLongArray) + { + for (int i = 0; i < col.size(); i++) + { + int encodedId = VarintUtil.readVarint(compressedIn); + col.set(i, FullDataPointUtil.encode(encodedId >> 2, 1, encodedId & 1, (byte) (encodedId & 2), (byte) 0)); + } + } + + // 3. height + for (LongArrayList col : outputDataLongArray) + { + for (int i = 0; i < col.size(); i++) + { + int height = VarintUtil.readVarint(compressedIn); + long data = col.getLong(i); + col.set(i, FullDataPointUtil.setHeight(data, height)); + } + } + + // 4. bottomY + int previousBottomY = 0; + for (LongArrayList col : outputDataLongArray) + { + for (int i = 0; i < col.size(); i++) + { + long data = col.getLong(i); + int error = 0; + if (FullDataPointUtil.getBottomY(data) != 0) + { + error = VarintUtil.zigzagDecode(VarintUtil.readVarint(compressedIn)); + } + int bottomY = previousBottomY - FullDataPointUtil.getHeight(data) + error; + col.set(i, FullDataPointUtil.setBottomY(data, bottomY)); + previousBottomY = bottomY; + } + } + + // 5. lights + for (LongArrayList col : outputDataLongArray) + { + for (int i = 0; i < col.size(); i++) + { + long data = col.getLong(i); + boolean hasLight = FullDataPointUtil.getBlockLight(data) != 0; + byte skyLight = 0; + byte blockLight = 0; + if (hasLight) + { + byte packedLight = compressedIn.readByte(); + skyLight = (byte) (packedLight & 0xF); + blockLight = (byte) (packedLight >> 4); + } + + col.set(i, FullDataPointUtil.setSkyLight( + FullDataPointUtil.setBlockLight(data, blockLight), + skyLight)); + } + } + + if (FullDataPointUtil.RUN_VALIDATION) + { + // These points all bypassed validation because of using setters. + for (LongArrayList col : outputDataLongArray) + { + for (int i = 0; i < col.size(); i++) + { + FullDataPointUtil.validateDatapoint(col.getLong(i)); + } + } + } + } + catch (EOFException e) + { + throw new DataCorruptedException(e); + } + } + private static void writeGenerationStepsToBlob(ByteArrayList inputColumnGenStepByteArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/VarintUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/VarintUtil.java new file mode 100644 index 000000000..ccad5cc17 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/util/VarintUtil.java @@ -0,0 +1,69 @@ +package com.seibel.distanthorizons.core.sql.dto.util; + +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; + +import java.io.IOException; + +public class VarintUtil +{ + + /** + * zigzagEncode maps 0=>0, -1=>1, 1=>2, -2=>3, 3=>4, etc. + * this helps encode small magnitude signed numbers as small varints. + * https://lemire.me/blog/2022/11/25/making-all-your-integers-positive-with-zigzag-encoding/ + */ + public static int zigzagEncode(int n) + { + // if n is (byte)-1, this results in: + // 0b1111_1110 ^ 0b1111_1111 == 0b0000_0001 + return (n << 1) ^ (n >> 31); + } + + public static int zigzagDecode(int n) + { return (n >>> 1) ^ -(n & 1); } + + + + /** + * @param value should be a zigzag encoded value + * created via {@link VarintUtil#zigzagEncode(int)} + */ + public static void writeVarint(DhDataOutputStream out, int value) throws IOException + { + if (value < 0) + { + throw new IllegalArgumentException("varint given ["+value+"], varint only accepts positive values."); + } + + while (value >= 128) + { + out.writeByte(value | 128); + value >>>= 7; // 128 = 2^7 + } + out.writeByte(value); + } + + public static int readVarint(DhDataInputStream in) throws IOException + { + int value = 0; + int shift = 0; + byte b; + do + { + if (shift >= 32) + { + throw new IOException("invalid varint"); + } + b = in.readByte(); + value |= (b & 127) << shift; + shift += 7; + } + while ((b & 128) != 0); + return value; + } + + + +} diff --git a/core/src/test/java/tests/DhFullDataSourceRepoTests.java b/core/src/test/java/tests/DhFullDataSourceRepoTests.java index 055933372..1f0bbd62e 100644 --- a/core/src/test/java/tests/DhFullDataSourceRepoTests.java +++ b/core/src/test/java/tests/DhFullDataSourceRepoTests.java @@ -87,16 +87,23 @@ public class DhFullDataSourceRepoTests Random seededRandom = new Random(3); - for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++) + for (int arrayIndex = 0; arrayIndex < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; arrayIndex++) { - fullDataArray[i] = new LongArrayList(1); + fullDataArray[arrayIndex] = new LongArrayList(1); // random column heights so we can differentiate // columns from each other int columnCount = Math.abs(seededRandom.nextInt() % 31) + 1; - for (int j = 0; j < columnCount; j++) + for (int colIndex = 0; colIndex < columnCount; colIndex++) { - fullDataArray[i].add(FullDataPointUtil.encode(j, 1, j, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT)); + long datapoint = FullDataPointUtil.encode( + colIndex, // id + 1, // height + colIndex, // relative min Y + (byte)(colIndex % LodUtil.MAX_MC_LIGHT), // block light + (byte)((colIndex + 2) % LodUtil.MAX_MC_LIGHT) // sky light + ); + fullDataArray[arrayIndex].add(datapoint); } } @@ -113,6 +120,18 @@ public class DhFullDataSourceRepoTests repo.save(originalDto); + // also create format-1 encoded version to ensure backwards compatibility + long posV1 = DhSectionPos.encode((byte) 6, 2, 3); + FullDataSourceV2 dataSourceFormatV1 = FullDataSourceV2.createWithData(posV1, dataMapping, fullDataArray, columnGenStep, columnWorldCompressionMode); + FullDataSourceV2DTO dtoFormatV1 = FullDataSourceV2DTO.CreateFromDataSource(dataSourceFormatV1, EDhApiDataCompressionMode.LZMA2); + FullDataSourceV2DTO.writeDataSourceDataArrayToBlobV1( + dataSourceFormatV1.dataPoints, + dtoFormatV1.compressedDataByteArray, + EDhApiDataCompressionMode.LZMA2); + dtoFormatV1.dataFormatVersion = FullDataSourceV2DTO.DATA_FORMAT.V1_NO_ADJACENT_DATA; + repo.save(dtoFormatV1); + + //=======================// // confirm DTO data is // @@ -152,6 +171,21 @@ public class DhFullDataSourceRepoTests } } + // check that we have proper backwards compatability to V1 + try (FullDataSourceV2 savedDataSource = repo.getByKey(posV1).createUnitTestDataSource()) + { + Assert.assertNotNull("Failed to create DataSource", savedDataSource); + assertArraysAreEqual(originalDataSource.columnGenerationSteps, savedDataSource.columnGenerationSteps); + assertArraysAreEqual(originalDataSource.columnWorldCompressionMode, + savedDataSource.columnWorldCompressionMode); + Assert.assertTrue(originalDataSource.dataPoints.length == savedDataSource.dataPoints.length); + + for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++) + { + assertArraysAreEqual(originalDataSource.dataPoints[i], savedDataSource.dataPoints[i]); + } + } + //==============// diff --git a/core/src/test/java/tests/VarintTest.java b/core/src/test/java/tests/VarintTest.java new file mode 100644 index 000000000..8970a5524 --- /dev/null +++ b/core/src/test/java/tests/VarintTest.java @@ -0,0 +1,94 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package tests; + +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.sql.dto.util.VarintUtil; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class VarintTest +{ + + @Test + public void Test() + { + Assert.assertEquals(0x80, 128); + + + // zig zag encoding is needed for varint handling, so test it first + for (int i = -256; i < 256; i++) + { + //testZigZagEncoding(i); + } + + for (int i = -256; i < 256; i++) + { + //testSingleVarint(i); + } + } + + private static void testZigZagEncoding(int value) + { + int encodedValue = VarintUtil.zigzagEncode(value); + int decodedValue = VarintUtil.zigzagDecode(encodedValue); + Assert.assertEquals(value, decodedValue); + } + + private static void testSingleVarint(int value) + { + // write to stream + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (DhDataOutputStream outputStream = new DhDataOutputStream(byteArrayOutputStream, EDhApiDataCompressionMode.UNCOMPRESSED)) + { + int encodedValue = VarintUtil.zigzagEncode(value); + VarintUtil.writeVarint(outputStream, encodedValue); // varint requires zig-zag encoding to function + } + catch (IOException e) + { + e.printStackTrace(); + Assert.fail("Fail writing varint ["+value+"], error: ["+e.getMessage()+"]"); + } + + + // read stream + byte[] byteArray = byteArrayOutputStream.toByteArray(); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + try (DhDataInputStream inputStream = new DhDataInputStream(byteArrayInputStream, EDhApiDataCompressionMode.UNCOMPRESSED)) + { + int encodedValue = VarintUtil.readVarint(inputStream); + int decodedValue = VarintUtil.zigzagDecode(encodedValue); + Assert.assertEquals(value, decodedValue); + } + catch (IOException e) + { + e.printStackTrace(); + Assert.fail("Fail reading varint ["+value+"], error: ["+e.getMessage()+"]"); + } + } + +} From 6fe0df7d0fd32c0a6e42343d6c1e104f4369726f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 13 Nov 2025 07:18:09 -0600 Subject: [PATCH 23/24] Don't duplicate adjacent data --- .../V2/FullDataSourceProviderV2.java | 2 +- .../distanthorizons/core/jar/JarMain.java | 2 +- .../core/sql/dto/FullDataSourceV2DTO.java | 415 +++++++++--------- .../core/sql/repo/FullDataSourceV2Repo.java | 53 --- .../java/tests/DhFullDataSourceRepoTests.java | 15 +- 5 files changed, 223 insertions(+), 264 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index 67c3401ad..22040a43b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -199,7 +199,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable @Nullable public FullDataSourceV2 get(long pos) { - try(FullDataSourceV2DTO dto = this.repo.getByPosNoAdj(pos)) + try(FullDataSourceV2DTO dto = this.repo.getByKey(pos)) { if (dto == null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java index f3ff831d3..806b3baeb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/JarMain.java @@ -239,7 +239,7 @@ public class JarMain private static void exportLodDataAtPosition(FullDataSourceV2Repo repo, File exportFile, long pos) { - FullDataSourceV2DTO dto = repo.getByPosNoAdj(pos); + FullDataSourceV2DTO dto = repo.getByKey(pos); if (dto == null) { LOGGER.error("Unable to find any data at the position ["+DhSectionPos.toString(pos)+"]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index d0446454f..1429a65aa 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -110,15 +110,15 @@ public class FullDataSourceV2DTO FullDataSourceV2DTO dto = FullDataSourceV2DTO.CreateEmptyDataSourceForDecoding(); // populate arrays - writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedDataByteArray, compressionModeEnum); + writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedDataByteArray, null, compressionModeEnum); writeGenerationStepsToBlob(dataSource.columnGenerationSteps, dto.compressedColumnGenStepByteArray, compressionModeEnum); writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, dto.compressedWorldCompressionModeByteArray, compressionModeEnum); writeDataMappingToBlob(dataSource.mapping, dto.compressedMappingByteArray, compressionModeEnum); // adjacent full data - writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedNorthAdjDataByteArray, EDhDirection.NORTH, compressionModeEnum); - writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedSouthAdjDataByteArray, EDhDirection.SOUTH, compressionModeEnum); - writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedEastAdjDataByteArray, EDhDirection.EAST, compressionModeEnum); - writeDataSourceAdjacentDataArrayToBlob(dataSource.dataPoints, dto.compressedWestAdjDataByteArray, EDhDirection.WEST, compressionModeEnum); + writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedNorthAdjDataByteArray, EDhDirection.NORTH, compressionModeEnum); + writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedSouthAdjDataByteArray, EDhDirection.SOUTH, compressionModeEnum); + writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedEastAdjDataByteArray, EDhDirection.EAST, compressionModeEnum); + writeDataSourceDataArrayToBlobV2(dataSource.dataPoints, dto.compressedWestAdjDataByteArray, EDhDirection.WEST, compressionModeEnum); // populate individual variables { @@ -140,7 +140,6 @@ public class FullDataSourceV2DTO /** Should only be used for subsequent decoding */ public static FullDataSourceV2DTO CreateEmptyDataSourceForDecoding() { return new FullDataSourceV2DTO(); } - private FullDataSourceV2DTO() { super(ARRAY_LIST_POOL, 8, 0, 0); @@ -231,6 +230,14 @@ public class FullDataSourceV2DTO // data // + // clear any old data so we can start fresh + for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++) + { + @NotNull LongArrayList array = dataSource.dataPoints[i]; + array.clear(); + array.add(FullDataPointUtil.EMPTY_DATA_POINT); + } + if (direction == null) { readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, dataSource.columnGenerationSteps, compressionModeEnum); @@ -242,7 +249,13 @@ public class FullDataSourceV2DTO } else { - readBlobToDataSourceDataArrayV2(this.compressedDataByteArray, dataSource.dataPoints, compressionModeEnum); + // doesn't include adjacent (ie edge) data + readBlobToDataSourceDataArrayV2(this.compressedDataByteArray, dataSource.dataPoints, null, compressionModeEnum); + + readBlobToDataSourceDataArrayV2(this.compressedNorthAdjDataByteArray, dataSource.dataPoints, EDhDirection.NORTH, compressionModeEnum); + readBlobToDataSourceDataArrayV2(this.compressedSouthAdjDataByteArray, dataSource.dataPoints, EDhDirection.SOUTH, compressionModeEnum); + readBlobToDataSourceDataArrayV2(this.compressedEastAdjDataByteArray, dataSource.dataPoints, EDhDirection.EAST, compressionModeEnum); + readBlobToDataSourceDataArrayV2(this.compressedWestAdjDataByteArray, dataSource.dataPoints, EDhDirection.WEST, compressionModeEnum); } } else @@ -252,7 +265,7 @@ public class FullDataSourceV2DTO // this is done so data sources down-stream // can all be handled identically regardless of // whether they're a full or partial data source - readDataSourceAdjacentDataArrayToBlob(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum); + readBlobToDataSourceDataArrayV2(this.compressedDataByteArray, dataSource.dataPoints, direction, compressionModeEnum); } @@ -305,108 +318,6 @@ public class FullDataSourceV2DTO // (de)serializing // //=================// - private static void writeDataSourceAdjacentDataArrayToBlob( - LongArrayList[] wholeInputDataArray, ByteArrayList outputByteArray, - EDhDirection direction, - EDhApiDataCompressionMode compressionModeEnum) throws IOException - { - // write the outputs to a stream to prep for writing to the database - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - - // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, - // but since this stream will be closed immediately after writing anyway, it won't be an issue - try (DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum)) - { - long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); - int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); - int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); - int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); - int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); - - for (int x = minX; x < maxX; x++) - { - for (int z = minZ; z < maxZ; z++) - { - int index = FullDataSourceV2.relativePosToIndex(x, z); - LongArrayList dataColumn = wholeInputDataArray[index]; - - // write column length - short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; - // a short is used instead of an int because at most we store 4096 vertical slices and a - // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) - compressedOut.writeShort(columnLength); - - // write column data (will be skipped if no data was present) - for (int y = 0; y < columnLength; y++) - { - compressedOut.writeLong(dataColumn.getLong(y)); - } - } - } - - - // generate the checksum (currently unused) - compressedOut.flush(); - byteArrayOutputStream.close(); - outputByteArray.addElements(0, byteArrayOutputStream.toByteArray()); - } - } - private static void readDataSourceAdjacentDataArrayToBlob( - @NotNull ByteArrayList inputCompressedDataByteArray, @NotNull LongArrayList[] outputDataLongArray, - @NotNull EDhDirection direction, - EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException - { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements()); - try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum)) - { - for (int i = 0; i < FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; i++) - { - @NotNull LongArrayList array = outputDataLongArray[i]; - array.clear(); - array.add(FullDataPointUtil.EMPTY_DATA_POINT); - } - - - long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); - int minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); - int maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); - int minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); - int maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); - - for (int x = minX; x < maxX; x++) - { - for (int z = minZ; z < maxZ; z++) - { - int index = FullDataSourceV2.relativePosToIndex(x, z); - LongArrayList dataColumn = outputDataLongArray[index]; - - // read the column length - short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later - if (dataColumnLength < 0) - { - throw new DataCorruptedException("Read DataSource adj[" + direction + "] Blob data at index [" + index + "], column length [" + dataColumnLength + "] should be greater than zero."); - } - - - ListUtil.clearAndSetSize(dataColumn, dataColumnLength); - - // read column data (will be skipped if no data was present) - for (int y = 0; y < dataColumnLength; y++) - { - long dataPoint = compressedIn.readLong(); - if (VALIDATE_INPUT_DATAPOINTS) - { - FullDataPointUtil.validateDatapoint(dataPoint); - } - dataColumn.set(y, dataPoint); - } - } - } - - } - } - - public static void writeDataSourceDataArrayToBlobV1( LongArrayList[] inputDataArray, ByteArrayList outputByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException @@ -481,100 +392,141 @@ public class FullDataSourceV2DTO private static void writeDataSourceDataArrayToBlobV2( LongArrayList[] inputDataArray, ByteArrayList outputByteArray, - EDhApiDataCompressionMode compressionModeEnum) throws IOException + @Nullable EDhDirection direction, EDhApiDataCompressionMode compressionModeEnum) throws IOException { + int minX, maxX, minZ, maxZ; + if (direction != null) + { + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + } + else + { + // skip the border data so we don't duplicate the adjacent data + minX = 1; + maxX = FullDataSourceV2.WIDTH-1; + minZ = 1; + maxZ = FullDataSourceV2.WIDTH-1; + } + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); try (DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum)) { - int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; - // this method would be simpler if we allocated a bunch of temporary arrays, // but we're trying to avoid garbage. // 1. column lengths - for (int xz = 0; xz < dataArrayLength; xz++) + for (int x = minX; x < maxX; x++) { - LongArrayList col = inputDataArray[xz]; - int size = col != null ? col.size() : 0; - VarintUtil.writeVarint(compressedOut, size); + for (int z = minZ; z < maxZ; z++) + { + int index = FullDataSourceV2.relativePosToIndex(x,z); + + LongArrayList col = inputDataArray[index]; + int size = (col != null) ? col.size() : 0; + VarintUtil.writeVarint(compressedOut, size); + } } // 2. column ids, with "is lit" and "is discontinuous" bits int previousBottomY = 0; - for (int xz = 0; xz < dataArrayLength; xz++) + for (int x = minX; x < maxX; x++) { - LongArrayList col = inputDataArray[xz]; - int size = col != null ? col.size() : 0; - for (int y = 0; y < size; y++) + for (int z = minZ; z < maxZ; z++) { - long data = col.getLong(y); + int index = FullDataSourceV2.relativePosToIndex(x, z); - int id = FullDataPointUtil.getId(data); - int height = FullDataPointUtil.getHeight(data); - int bottomY = FullDataPointUtil.getBottomY(data); - - boolean hasLight = (FullDataPointUtil.getBlockLight(data) | FullDataPointUtil.getSkyLight(data)) != LodUtil.MIN_MC_LIGHT; - - // all datapoints are contiguous, with no gaps - // so having both height and bottomY is redundant. We could store the prediction - // in an array, but it's much cheaper to just recompute it later. - int expectedBottomY = previousBottomY - height; - boolean hasDiscontinuity = bottomY != expectedBottomY; - previousBottomY = bottomY; - - VarintUtil.writeVarint(compressedOut, (id << 2) | (hasLight ? 2 : 0) | (hasDiscontinuity ? 1 : 0)); + LongArrayList col = inputDataArray[index]; + int size = col != null ? col.size() : 0; + for (int y = 0; y < size; y++) + { + long data = col.getLong(y); + + int id = FullDataPointUtil.getId(data); + int height = FullDataPointUtil.getHeight(data); + int bottomY = FullDataPointUtil.getBottomY(data); + + boolean hasLight = (FullDataPointUtil.getBlockLight(data) | FullDataPointUtil.getSkyLight(data)) != LodUtil.MIN_MC_LIGHT; + + // all datapoints are contiguous, with no gaps + // so having both height and bottomY is redundant. We could store the prediction + // in an array, but it's much cheaper to just recompute it later. + int expectedBottomY = previousBottomY - height; + boolean hasDiscontinuity = bottomY != expectedBottomY; + previousBottomY = bottomY; + + VarintUtil.writeVarint(compressedOut, (id << 2) | (hasLight ? 2 : 0) | (hasDiscontinuity ? 1 : 0)); + } } } // 3. heights - for (int xz = 0; xz < dataArrayLength; xz++) + for (int x = minX; x < maxX; x++) { - LongArrayList col = inputDataArray[xz]; - int size = (col != null) ? col.size() : 0; - for (int y = 0; y < size; y++) + for (int z = minZ; z < maxZ; z++) { - long data = col.getLong(y); - VarintUtil.writeVarint(compressedOut, FullDataPointUtil.getHeight(data)); + int index = FullDataSourceV2.relativePosToIndex(x, z); + + LongArrayList col = inputDataArray[index]; + int size = (col != null) ? col.size() : 0; + for (int y = 0; y < size; y++) + { + long data = col.getLong(y); + VarintUtil.writeVarint(compressedOut, FullDataPointUtil.getHeight(data)); + } } } // 4. bottomY (only the mis-predicted ones) previousBottomY = 0; - for (int xz = 0; xz < dataArrayLength; xz++) + for (int x = minX; x < maxX; x++) { - LongArrayList col = inputDataArray[xz]; - int size = (col != null) ? col.size() : 0; - for (int y = 0; y < size; y++) + for (int z = minZ; z < maxZ; z++) { - long data = col.getLong(y); + int index = FullDataSourceV2.relativePosToIndex(x, z); - int height = FullDataPointUtil.getHeight(data); - int bottomY = FullDataPointUtil.getBottomY(data); - - int expectedBottomY = previousBottomY - height; - if (bottomY != expectedBottomY) + LongArrayList col = inputDataArray[index]; + int size = (col != null) ? col.size() : 0; + for (int y = 0; y < size; y++) { - VarintUtil.writeVarint(compressedOut, VarintUtil.zigzagEncode(bottomY - expectedBottomY)); + long data = col.getLong(y); + + int height = FullDataPointUtil.getHeight(data); + int bottomY = FullDataPointUtil.getBottomY(data); + + int expectedBottomY = previousBottomY - height; + if (bottomY != expectedBottomY) + { + VarintUtil.writeVarint(compressedOut, VarintUtil.zigzagEncode(bottomY - expectedBottomY)); + } + previousBottomY = bottomY; } - previousBottomY = bottomY; } } // 5. packed Light (only lit sections) - for (int xz = 0; xz < dataArrayLength; xz++) + for (int x = minX; x < maxX; x++) { - LongArrayList col = inputDataArray[xz]; - int size = col != null ? col.size() : 0; - for (int y = 0; y < size; y++) + for (int z = minZ; z < maxZ; z++) { - long data = col.getLong(y); - int blockLight = FullDataPointUtil.getBlockLight(data); - int skyLight = FullDataPointUtil.getSkyLight(data); - byte packedLight = (byte) ((blockLight << 4) | skyLight); - if (packedLight != 0) + int index = FullDataSourceV2.relativePosToIndex(x, z); + + LongArrayList col = inputDataArray[index]; + int size = (col != null) ? col.size() : 0; + for (int y = 0; y < size; y++) { - compressedOut.writeByte(packedLight); + long data = col.getLong(y); + int blockLight = FullDataPointUtil.getBlockLight(data); + int skyLight = FullDataPointUtil.getSkyLight(data); + byte packedLight = (byte) ((blockLight << 4) | skyLight); + if (packedLight != 0) + { + compressedOut.writeByte(packedLight); + } } } } @@ -586,89 +538,142 @@ public class FullDataSourceV2DTO } private static void readBlobToDataSourceDataArrayV2( ByteArrayList inputCompressedDataByteArray, - LongArrayList[] outputDataLongArray, EDhApiDataCompressionMode compressionModeEnum) + LongArrayList[] outputDataLongArray, + @Nullable EDhDirection direction, EDhApiDataCompressionMode compressionModeEnum) throws IOException, DataCorruptedException { + int minX, maxX, minZ, maxZ; + if (direction != null) + { + long encodedMinMaxPos = FullDataMinMaxPosUtil.getEncodedMinMaxPos(direction); + minX = FullDataMinMaxPosUtil.getAdjMinX(encodedMinMaxPos); + maxX = FullDataMinMaxPosUtil.getAdjMaxX(encodedMinMaxPos); + minZ = FullDataMinMaxPosUtil.getAdjMinZ(encodedMinMaxPos); + maxZ = FullDataMinMaxPosUtil.getAdjMaxZ(encodedMinMaxPos); + } + else + { + // skip the border data so we don't duplicate the adjacent data + minX = 1; + maxX = FullDataSourceV2.WIDTH-1; + minZ = 1; + maxZ = FullDataSourceV2.WIDTH-1; + } + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(inputCompressedDataByteArray.elements()); try (DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum)) { // 1. column counts, preallocate - int numColumns = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; - for (int i = 0; i < numColumns; i++) + for (int x = minX; x < maxX; x++) { - int count = VarintUtil.readVarint(compressedIn); - ListUtil.clearAndSetSize(outputDataLongArray[i], count); + for (int z = minZ; z < maxZ; z++) + { + int index = FullDataSourceV2.relativePosToIndex(x, z); + + int count = VarintUtil.readVarint(compressedIn); + ListUtil.clearAndSetSize(outputDataLongArray[index], count); + } } // 2. ids and flags for min_y and light - for (LongArrayList col : outputDataLongArray) + for (int x = minX; x < maxX; x++) { - for (int i = 0; i < col.size(); i++) + for (int z = minZ; z < maxZ; z++) { - int encodedId = VarintUtil.readVarint(compressedIn); - col.set(i, FullDataPointUtil.encode(encodedId >> 2, 1, encodedId & 1, (byte) (encodedId & 2), (byte) 0)); + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList col = outputDataLongArray[index]; + + for (int i = 0; i < col.size(); i++) + { + int encodedId = VarintUtil.readVarint(compressedIn); + col.set(i, FullDataPointUtil.encode(encodedId >> 2, 1, encodedId & 1, (byte) (encodedId & 2), (byte) 0)); + } } } // 3. height - for (LongArrayList col : outputDataLongArray) + for (int x = minX; x < maxX; x++) { - for (int i = 0; i < col.size(); i++) + for (int z = minZ; z < maxZ; z++) { - int height = VarintUtil.readVarint(compressedIn); - long data = col.getLong(i); - col.set(i, FullDataPointUtil.setHeight(data, height)); + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList col = outputDataLongArray[index]; + + for (int i = 0; i < col.size(); i++) + { + int height = VarintUtil.readVarint(compressedIn); + long data = col.getLong(i); + col.set(i, FullDataPointUtil.setHeight(data, height)); + } } } // 4. bottomY int previousBottomY = 0; - for (LongArrayList col : outputDataLongArray) + for (int x = minX; x < maxX; x++) { - for (int i = 0; i < col.size(); i++) + for (int z = minZ; z < maxZ; z++) { - long data = col.getLong(i); - int error = 0; - if (FullDataPointUtil.getBottomY(data) != 0) + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList col = outputDataLongArray[index]; + + for (int i = 0; i < col.size(); i++) { - error = VarintUtil.zigzagDecode(VarintUtil.readVarint(compressedIn)); + long data = col.getLong(i); + int error = 0; + if (FullDataPointUtil.getBottomY(data) != 0) + { + error = VarintUtil.zigzagDecode(VarintUtil.readVarint(compressedIn)); + } + int bottomY = previousBottomY - FullDataPointUtil.getHeight(data) + error; + col.set(i, FullDataPointUtil.setBottomY(data, bottomY)); + previousBottomY = bottomY; } - int bottomY = previousBottomY - FullDataPointUtil.getHeight(data) + error; - col.set(i, FullDataPointUtil.setBottomY(data, bottomY)); - previousBottomY = bottomY; } } // 5. lights - for (LongArrayList col : outputDataLongArray) + for (int x = minX; x < maxX; x++) { - for (int i = 0; i < col.size(); i++) + for (int z = minZ; z < maxZ; z++) { - long data = col.getLong(i); - boolean hasLight = FullDataPointUtil.getBlockLight(data) != 0; - byte skyLight = 0; - byte blockLight = 0; - if (hasLight) - { - byte packedLight = compressedIn.readByte(); - skyLight = (byte) (packedLight & 0xF); - blockLight = (byte) (packedLight >> 4); - } + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList col = outputDataLongArray[index]; - col.set(i, FullDataPointUtil.setSkyLight( - FullDataPointUtil.setBlockLight(data, blockLight), - skyLight)); + for (int i = 0; i < col.size(); i++) + { + long data = col.getLong(i); + boolean hasLight = FullDataPointUtil.getBlockLight(data) != 0; + byte skyLight = 0; + byte blockLight = 0; + if (hasLight) + { + byte packedLight = compressedIn.readByte(); + skyLight = (byte) (packedLight & 0xF); + blockLight = (byte) (packedLight >> 4); + } + + col.set(i, FullDataPointUtil.setSkyLight( + FullDataPointUtil.setBlockLight(data, blockLight), + skyLight)); + } } } if (FullDataPointUtil.RUN_VALIDATION) { // These points all bypassed validation because of using setters. - for (LongArrayList col : outputDataLongArray) + for (int x = minX; x < maxX; x++) { - for (int i = 0; i < col.size(); i++) + for (int z = minZ; z < maxZ; z++) { - FullDataPointUtil.validateDatapoint(col.getLong(i)); + int index = FullDataSourceV2.relativePosToIndex(x, z); + LongArrayList col = outputDataLongArray[index]; + + for (int i = 0; i < col.size(); i++) + { + FullDataPointUtil.validateDatapoint(col.getLong(i)); + } } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 5ca9c302c..f16bbadce 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -344,59 +344,6 @@ public class FullDataSourceV2Repo extends AbstractDhRepo Date: Fri, 14 Nov 2025 07:46:02 -0600 Subject: [PATCH 24/24] Speed up shutdown and reduce logging --- .../V2/FullDataSourceProviderV2.java | 65 ++++++++++--------- .../V2/FullDataUpdatePropagatorV2.java | 25 +++---- .../fullDatafile/V2/FullDataUpdaterV2.java | 21 ++++-- .../core/generation/WorldGenerationQueue.java | 18 ++--- .../core/level/DhServerLevel.java | 1 - .../core/sql/repo/AbstractDhRepo.java | 2 +- .../util/threading/PriorityTaskPicker.java | 4 -- .../core/world/AbstractDhServerWorld.java | 23 ++++++- .../core/world/DhClientServerWorld.java | 24 ++++++- .../core/world/DhClientWorld.java | 25 +++++-- .../core/world/DhServerWorld.java | 2 +- 11 files changed, 129 insertions(+), 81 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java index 22040a43b..5c2207212 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataSourceProviderV2.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; /** @@ -76,14 +77,11 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable public static final byte LEAF_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - - protected final ReentrantLock closeLock = new ReentrantLock(); - protected volatile boolean isShutdown = false; - - protected final File saveDir; - public final FullDataSourceV2Repo repo; + + protected final AtomicBoolean isShutdownRef = new AtomicBoolean(false); + protected final File saveDir; protected final IDhLevel level; protected final String levelId; @@ -174,6 +172,11 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable */ public CompletableFuture getAsync(long pos) { + if (this.isShutdownRef.get()) + { + return CompletableFuture.completedFuture(null); + } + AbstractExecutorService executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { @@ -199,6 +202,11 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable @Nullable public FullDataSourceV2 get(long pos) { + if (this.isShutdownRef.get()) + { + return null; + } + try(FullDataSourceV2DTO dto = this.repo.getByKey(pos)) { if (dto == null) @@ -267,6 +275,11 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable */ public FullDataSourceV2 getAdjForDirection(long pos, EDhDirection direction) { + if (this.isShutdownRef.get()) + { + return null; + } + try(FullDataSourceV2DTO dto = this.repo.getAdjByPosAndDirection(pos, direction)) { if (dto == null) @@ -386,7 +399,14 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable @Nullable public Long getTimestampForPos(long pos) - { return this.repo.getTimestampForPos(pos); } + { + if (this.isShutdownRef.get()) + { + return null; + } + + return this.repo.getTimestampForPos(pos); + } @@ -416,28 +436,15 @@ public class FullDataSourceProviderV2 implements IDebugRenderable, AutoCloseable @Override public void close() { - try - { - LOGGER.debug("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); - - this.closeLock.lock(); - this.isShutdown = true; - - this.dataUpdater.close(); - this.updatePropagator.close(); - this.dataMigratorV1.close(); - - // wait a moment so any queued saves can finish queuing, - // otherwise we might not see everything that needs saving and attempt to use a closed repo - Thread.sleep(200); - - this.repo.close(); - } - catch (InterruptedException ignore) { } - finally - { - this.closeLock.unlock(); - } + LOGGER.debug("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); + + this.isShutdownRef.set(true); + + this.dataUpdater.close(); + this.updatePropagator.close(); + this.dataMigratorV1.close(); + + this.repo.close(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java index 0d783cdbe..7bc8698d7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdatePropagatorV2.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseable @@ -41,9 +42,6 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab { return NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Common.MultiThreading.numberOfThreads.get(); } - private final FullDataSourceProviderV2 provider; - private final FullDataUpdaterV2 dataUpdater; - /** * Tracks which positions are currently being updated @@ -59,9 +57,14 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab @Nullable public final ThreadPoolExecutor updateQueueProcessor; + private final AtomicBoolean isShutdownRef = new AtomicBoolean(false); private final String levelId; + private final FullDataSourceProviderV2 provider; + private final FullDataUpdaterV2 dataUpdater; + + //=============// // constructor // @@ -125,8 +128,6 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e); } } - - LOGGER.info("Update thread ["+Thread.currentThread().getName()+"] terminated."); } /** will always apply updates */ private void runParentUpdates(PriorityTaskPicker.Executor executor, DhBlockPos targetBlockPos) @@ -387,20 +388,10 @@ public class FullDataUpdatePropagatorV2 implements IDebugRenderable, AutoCloseab @Override public void close() { - try + if (this.updateQueueProcessor != null) { - //LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); - - if (this.updateQueueProcessor != null) - { - this.updateQueueProcessor.shutdownNow(); - } - - // wait a moment so any queued saves can finish queuing, - // otherwise we might not see everything that needs saving and attempt to use a closed repo - Thread.sleep(200); + this.updateQueueProcessor.shutdownNow(); } - catch (InterruptedException ignore) { } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java index eefb76c3b..50246c292 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/V2/FullDataUpdaterV2.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Set; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; @@ -26,10 +27,6 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable { private static final DhLogger LOGGER = new DhLoggerBuilder().build(); - - private final FullDataSourceProviderV2 provider; - - protected final PositionalLockProvider updateLockProvider = new PositionalLockProvider(); /** * generally just used for debugging, @@ -41,6 +38,9 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); private final String levelId; + private final AtomicBoolean isShutdownRef = new AtomicBoolean(false); + + private final FullDataSourceProviderV2 provider; @@ -67,6 +67,11 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable */ public CompletableFuture updateDataSourceAsync(@NotNull FullDataSourceV2 inputDataSource) { + if (this.isShutdownRef.get()) + { + return CompletableFuture.completedFuture(null); + } + AbstractExecutorService executor = ThreadPoolUtil.getChunkToLodBuilderExecutor(); if (executor == null || executor.isTerminated()) { @@ -104,6 +109,12 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable /** After this method returns the inputData will be written to file. */ public void updateDataSource(@NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos) { + if (this.isShutdownRef.get()) + { + return; + } + + long updatePos = inputData.getPos(); boolean methodLocked = false; @@ -231,7 +242,7 @@ public class FullDataUpdaterV2 implements IDebugRenderable, AutoCloseable @Override public void close() { - //LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.levelId + "]."); + this.isShutdownRef.set(true); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index bc502a521..1e668bc3f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -621,19 +621,13 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb LodUtil.assertTrue(this.generatorClosingFuture != null); - LOGGER.info("Awaiting world generator thread pool termination..."); - try + LOGGER.info("Shutting down world generator thread pool..."); + + AbstractExecutorService executor = ThreadPoolUtil.getWorldGenExecutor(); + if (executor != null) { - int waitTimeInSeconds = 3; - AbstractExecutorService executor = ThreadPoolUtil.getWorldGenExecutor(); - if (executor != null && !executor.awaitTermination(waitTimeInSeconds, TimeUnit.SECONDS)) - { - LOGGER.warn("World generator thread pool shutdown didn't complete after [" + waitTimeInSeconds + "] seconds. Some world generator requests may still be running."); - } - } - catch (InterruptedException e) - { - LOGGER.warn("World generator thread pool shutdown interrupted! Ignoring child threads...", e); + List tasks = executor.shutdownNow(); + LOGGER.info("World generator thread pool shutdown with [" + tasks.size() + "] incomplete tasks."); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index e230f09c7..2539acd5d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -107,7 +107,6 @@ public class DhServerLevel extends AbstractDhServerLevel { super.close(); this.serverside.close(); - LOGGER.info("Closed DHLevel for ["+this.getLevelWrapper()+"]."); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index 76bcfb536..c015ac836 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -369,7 +369,7 @@ public abstract class AbstractDhRepo> implemen if (DbConnectionClosedException.isClosedException(e)) { - throw new DbConnectionClosedException(e); + return new ArrayList<>(); } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/PriorityTaskPicker.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/PriorityTaskPicker.java index c2ed4adae..e55fa8679 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/PriorityTaskPicker.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/PriorityTaskPicker.java @@ -98,10 +98,6 @@ public class PriorityTaskPicker // Clear this executor's tasks since we no longer expect anything to execute. executor.taskQueue.clear(); } - else - { - throw e; - } } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/AbstractDhServerWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/AbstractDhServerWorld.java index 68384b1be..9d2a160ca 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/AbstractDhServerWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/AbstractDhServerWorld.java @@ -10,8 +10,10 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; import org.jetbrains.annotations.NotNull; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; public abstract class AbstractDhServerWorld extends AbstractDhWorld implements IDhServerWorld @@ -134,10 +136,9 @@ public abstract class AbstractDhServerWorld> closeFutures = new ArrayList<>(); for (TDhServerLevel level : this.dhLevelByLevelWrapper.values()) { - LOGGER.info("Unloading level [" + level.getLevelWrapper().getDhIdentifier() + "]."); - // level wrapper shouldn't be null, but just in case IServerLevelWrapper serverLevelWrapper = level.getServerLevelWrapper(); if (serverLevelWrapper != null) @@ -145,7 +146,23 @@ public abstract class AbstractDhServerWorld closeFuture = new CompletableFuture<>(); + Thread closeThread = new Thread(() -> + { + level.close(); + closeFuture.complete(null); + }, "level shutdown"); + closeThread.start(); + closeFutures.add(closeFuture); + } + + // wait for all the levels to finish closing + for (CompletableFuture future : closeFutures) + { + future.join(); } this.dhLevelByLevelWrapper.clear(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java index a0dc1ab30..ce9693d0a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java @@ -28,9 +28,11 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; import org.jetbrains.annotations.NotNull; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; public class DhClientServerWorld extends AbstractDhServerWorld implements IDhClientWorld @@ -143,13 +145,13 @@ public class DhClientServerWorld extends AbstractDhServerWorld> closeFutures = new ArrayList<>(); + synchronized (this.dhLevels) { // close each level for (DhClientServerLevel level : this.dhLevels) { - LOGGER.info("Unloading level [" + level.getServerLevelWrapper().getDhIdentifier() + "]."); - // level wrapper shouldn't be null, but just in case IServerLevelWrapper serverLevelWrapper = level.getServerLevelWrapper(); if (serverLevelWrapper != null) @@ -157,10 +159,26 @@ public class DhClientServerWorld extends AbstractDhServerWorld closeFuture = new CompletableFuture<>(); + Thread closeThread = new Thread(() -> + { + level.close(); + closeFuture.complete(null); + }, "level shutdown"); + closeThread.start(); + closeFutures.add(closeFuture); } } + // wait for all the levels to finish closing + for (CompletableFuture future : closeFutures) + { + future.join(); + } + + this.dhLevelByLevelWrapper.clear(); this.eventLoop.close(); LOGGER.info("Closed DhWorld of type " + this.environment); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java index 624c5d82d..7ed347e80 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientWorld.java @@ -29,7 +29,9 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapp import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import org.jetbrains.annotations.NotNull; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -127,14 +129,11 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld public void close() { this.networkState.close(); - this.dhTickerThread.shutdownNow(); - + ArrayList> closeFutures = new ArrayList<>(); for (DhClientLevel dhClientLevel : this.levels.values()) { - LOGGER.info("Unloading level [" + dhClientLevel.getLevelWrapper().getDhIdentifier() + "]."); - // level wrapper shouldn't be null, but just in case IClientLevelWrapper clientLevelWrapper = dhClientLevel.getClientLevelWrapper(); if (clientLevelWrapper != null) @@ -142,7 +141,23 @@ public class DhClientWorld extends AbstractDhWorld implements IDhClientWorld clientLevelWrapper.onUnload(); } - dhClientLevel.close(); + + // close levels asynchronously to speed up + // shutdown on servers with a lot of levels + CompletableFuture closeFuture = new CompletableFuture<>(); + Thread closeThread = new Thread(() -> + { + dhClientLevel.close(); + closeFuture.complete(null); + }, "level shutdown"); + closeThread.start(); + closeFutures.add(closeFuture); + } + + // wait for all the levels to finish closing + for (CompletableFuture future : closeFutures) + { + future.join(); } this.levels.clear(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java index d638f3b5a..9871be809 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhServerWorld.java @@ -64,7 +64,7 @@ public class DhServerWorld extends AbstractDhServerWorld if (this.dhLevelByLevelWrapper.containsKey(wrapper)) { - LOGGER.info("Unloading level {} ", this.dhLevelByLevelWrapper.get(wrapper)); + DhServerLevel level = this.dhLevelByLevelWrapper.get(wrapper); wrapper.onUnload(); this.dhLevelByLevelWrapper.remove(wrapper).close(); }